version 0.3.31
[fms.git] / src / freenet / messagelistrequester.cpp
1 #include "../../include/freenet/messagelistrequester.h"\r
2 #include "../../include/freenet/messagelistxml.h"\r
3 \r
4 #include <Poco/DateTimeFormatter.h>\r
5 #include <Poco/DateTimeParser.h>\r
6 #include <Poco/Timestamp.h>\r
7 \r
8 #ifdef XMEM\r
9         #include <xmem.h>\r
10 #endif\r
11 \r
12 MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)\r
13 {\r
14         Initialize();\r
15 }\r
16 \r
17 MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)\r
18 {\r
19         Initialize();\r
20 }\r
21 \r
22 const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) const\r
23 {\r
24         std::vector<std::string> dateparts;\r
25         int year=0;\r
26         int month=0;\r
27         int day=0;\r
28         Poco::DateTime today;\r
29 \r
30         StringFunctions::Split(datestr,"-",dateparts);\r
31         if(dateparts.size()==3)\r
32         {\r
33                 StringFunctions::Convert(dateparts[0],year);\r
34                 StringFunctions::Convert(dateparts[1],month);\r
35                 StringFunctions::Convert(dateparts[2],day);\r
36                 if(today.year()>year || (today.year()==year && today.month()>month) || (today.year()==year && today.month()==month && today.day()>=day))\r
37                 {\r
38                         return true;\r
39                 }\r
40                 else\r
41                 {\r
42                         return false;\r
43                 }\r
44         }\r
45         else\r
46         {\r
47                 return false;\r
48         }\r
49 \r
50 }\r
51 \r
52 const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &datestr) const\r
53 {\r
54         Poco::DateTime checkdate;\r
55         Poco::DateTime date;\r
56         int tzdiff=0;\r
57         if(Poco::DateTimeParser::tryParse(datestr,date,tzdiff))\r
58         {\r
59                 checkdate-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);\r
60                 if(checkdate<=date)\r
61                 {\r
62                         return true;\r
63                 }\r
64                 else\r
65                 {\r
66                         return false;\r
67                 }\r
68         }\r
69         else\r
70         {\r
71                 return false;\r
72         }\r
73 }\r
74 \r
75 void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards, const bool forceload)\r
76 {\r
77         // only query database when forced, or an 30 minutes have passed since last query\r
78         if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime())\r
79         {\r
80                 m_boardscache.clear();\r
81                 SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
82                 st.Step();\r
83                 while(st.RowReturned())\r
84                 {\r
85                         std::string boardname="";\r
86                         std::string tempval="";\r
87                         st.ResultText(0,boardname);\r
88                         st.ResultText(1,tempval);\r
89 \r
90                         if(tempval=="true")\r
91                         {\r
92                                 m_boardscache[boardname]=true;\r
93                         }\r
94                         else\r
95                         {\r
96                                 m_boardscache[boardname]=false;\r
97                         }\r
98 \r
99                         st.Step();\r
100                 }\r
101                 m_boardscacheupdate=Poco::DateTime();\r
102         }\r
103 \r
104         boards=m_boardscache;\r
105 \r
106 }\r
107 \r
108 const bool MessageListRequester::HandleAllData(FCPv2::Message &message)\r
109 {       \r
110         SQLite3DB::Statement st;\r
111         std::vector<std::string> idparts;\r
112         long datalength;\r
113         std::vector<char> data;\r
114         MessageListXML xml;\r
115         long identityid;\r
116         long fromidentityid;\r
117         long index;\r
118         std::map<std::string,bool> boards;      // list of boards and if we will save messages for that board or not\r
119         std::map<std::string,long> identityids; // list of identity public keys and their id in the database\r
120         bool addmessage=false;\r
121         std::string boardsstr="";\r
122         std::string datestr="";\r
123         std::vector<std::string> dateparts;\r
124 \r
125         GetBoardList(boards);\r
126 \r
127         StringFunctions::Split(message["Identifier"],"|",idparts);\r
128         StringFunctions::Convert(message["DataLength"],datalength);\r
129         StringFunctions::Convert(idparts[1],identityid);\r
130         StringFunctions::Convert(idparts[2],index);\r
131 \r
132         fromidentityid=identityid;\r
133 \r
134         // wait for all data to be received from connection\r
135         m_fcp->WaitForBytes(1000,datalength);\r
136 \r
137         // if we got disconnected- return immediately\r
138         if(m_fcp->IsConnected()==false)\r
139         {\r
140                 return false;\r
141         }\r
142 \r
143         // receive the file\r
144         m_fcp->Receive(data,datalength);\r
145 \r
146         // parse file into xml and update the database\r
147         if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
148         {\r
149 \r
150                 m_db->Execute("BEGIN;");\r
151 \r
152                 SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
153                 SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);");\r
154                 SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
155 \r
156                 for(long i=0; i<xml.MessageCount(); i++)\r
157                 {\r
158 \r
159                         // go through each board the message was posted to and see if we are saving messages to that board\r
160                         // if the board isn't found, see if we are saving messages to new boards\r
161                         boardsstr="";\r
162                         addmessage=false;\r
163                         std::vector<std::string> messageboards=xml.GetBoards(i);\r
164                         for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
165                         {\r
166                                 if(boards.find((*j))!=boards.end())\r
167                                 {\r
168                                         if(boards[(*j)]==true)\r
169                                         {\r
170                                                 addmessage=true;\r
171                                         }\r
172                                 }\r
173                                 else if(m_savetonewboards==true)\r
174                                 {\r
175                                         addmessage=true;\r
176                                 }\r
177                                 if(j!=messageboards.begin())\r
178                                 {\r
179                                         boardsstr+=", ";\r
180                                 }\r
181                                 boardsstr+=(*j);\r
182                         }\r
183 \r
184                         if(CheckDateNotFuture(xml.GetDate(i))==false)\r
185                         {\r
186                                 addmessage=false;\r
187                                 m_log->error(m_fcpuniquename+"::HandleAllData date for message is in future! "+xml.GetDate(i));\r
188                         }\r
189 \r
190                         if(addmessage==true && CheckDateWithinMaxDays(xml.GetDate(i))==false)\r
191                         {\r
192                                 addmessage=false;\r
193                         }\r
194 \r
195                         if(addmessage==true)\r
196                         {\r
197                                 mst.Bind(0,identityid);\r
198                                 mst.Bind(1,xml.GetDate(i));\r
199                                 mst.Bind(2,xml.GetIndex(i));\r
200                                 mst.Bind(3,identityid);\r
201                                 mst.Step();\r
202                                 mst.Reset();\r
203 \r
204                                 // We need to update ID here, in case this index was already inserted from another\r
205                                 // identity's message list.  This doesn't reset try count - maybe we should if the from\r
206                                 // identity was another identity\r
207                                 ust.Bind(0,identityid);\r
208                                 ust.Bind(1,identityid);\r
209                                 ust.Bind(2,xml.GetDate(i));\r
210                                 ust.Bind(3,xml.GetIndex(i));\r
211                                 ust.Step();\r
212                                 ust.Reset();\r
213 \r
214                                 m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i));\r
215 \r
216                         }\r
217                         else\r
218                         {\r
219                                 //m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr+" on "+xml.GetDate(i));\r
220                         }\r
221                 }\r
222 \r
223                 // insert external message indexes\r
224                 for(long i=0; i<xml.ExternalMessageCount(); i++)\r
225                 {\r
226                         if(xml.GetExternalType(i)=="Keyed")\r
227                         {\r
228                                 // go through each board the message was posted to and see if we are saving messages to that board\r
229                                 // if the board isn't found, see if we are saving messages to new boards\r
230                                 boardsstr="";\r
231                                 addmessage=false;\r
232                                 std::vector<std::string> messageboards=xml.GetExternalBoards(i);\r
233                                 for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
234                                 {\r
235                                         if(boards.find((*j))!=boards.end())\r
236                                         {\r
237                                                 if(boards[(*j)]==true)\r
238                                                 {\r
239                                                         addmessage=true;\r
240                                                 }\r
241                                         }\r
242                                         else if(m_savetonewboards==true)\r
243                                         {\r
244                                                 addmessage=true;\r
245                                         }\r
246                                         if(j!=messageboards.begin())\r
247                                         {\r
248                                                 boardsstr+=", ";\r
249                                         }\r
250                                         boardsstr+=(*j);\r
251                                 }\r
252 \r
253                                 if(CheckDateNotFuture(xml.GetExternalDate(i))==false)\r
254                                 {\r
255                                         addmessage=false;\r
256                                         m_log->error(m_fcpuniquename+"::HandleAllData date for external message is in future! "+xml.GetExternalDate(i));\r
257                                 }\r
258 \r
259                                 if(addmessage==true && CheckDateWithinMaxDays(xml.GetExternalDate(i))==false)\r
260                                 {\r
261                                         addmessage=false;\r
262                                 }\r
263 \r
264                                 if(addmessage==true)\r
265                                 {\r
266                                         int thisidentityid=0;\r
267                                         if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end())\r
268                                         {\r
269                                                 thisidentityid=identityids[xml.GetExternalIdentity(i)];\r
270                                         }\r
271                                         else\r
272                                         {\r
273                                                 spk.Bind(0,xml.GetExternalIdentity(i));\r
274                                                 spk.Step();\r
275 \r
276                                                 if(spk.RowReturned())\r
277                                                 {\r
278                                                         spk.ResultInt(0,thisidentityid);\r
279                                                         identityids[xml.GetExternalIdentity(i)]=thisidentityid;\r
280                                                 }\r
281 \r
282                                                 spk.Reset();\r
283                                         }\r
284 \r
285                                         if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end())\r
286                                         {\r
287                                                 mst.Bind(0,thisidentityid);\r
288                                                 mst.Bind(1,xml.GetExternalDate(i));\r
289                                                 mst.Bind(2,xml.GetExternalIndex(i));\r
290                                                 mst.Bind(3,fromidentityid);\r
291                                                 mst.Step();\r
292                                                 mst.Reset();\r
293 \r
294                                                 m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i));\r
295                                         }\r
296                                 }\r
297                                 else\r
298                                 {\r
299                                         //m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i) + " on " + xml.GetExternalDate(i));\r
300                                 }\r
301                         }\r
302                 }\r
303 \r
304                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");\r
305                 st.Bind(0,identityid);\r
306                 st.Bind(1,idparts[4]);\r
307                 st.Bind(2,index);\r
308                 st.Step();\r
309                 st.Finalize();\r
310 \r
311                 spk.Finalize();\r
312                 mst.Finalize();\r
313                 ust.Finalize();\r
314 \r
315                 m_db->Execute("COMMIT;");\r
316 \r
317                 m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
318         }\r
319         else\r
320         {\r
321                 // bad data - mark index\r
322                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
323                 st.Bind(0,identityid);\r
324                 st.Bind(1,idparts[4]);\r
325                 st.Bind(2,index);\r
326                 st.Step();\r
327                 st.Finalize();\r
328 \r
329                 m_log->error(m_fcpuniquename+"::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);\r
330         }\r
331 \r
332         // remove this identityid from request list\r
333         RemoveFromRequestList(identityid);\r
334 \r
335         // keep 2 days of request indexes in the cache\r
336         while(m_requestindexcache.size()>2)\r
337         {\r
338                 m_requestindexcache.erase(m_requestindexcache.begin());\r
339         }\r
340 \r
341         return true;\r
342 \r
343 }\r
344 \r
345 const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)\r
346 {\r
347         SQLite3DB::Statement st;\r
348         std::vector<std::string> idparts;\r
349         long identityid;\r
350         long index;\r
351 \r
352         StringFunctions::Split(message["Identifier"],"|",idparts);\r
353         StringFunctions::Convert(idparts[1],identityid);\r
354         StringFunctions::Convert(idparts[2],index);     \r
355 \r
356         // code 27 - permanent redirect\r
357         if(message["Code"]=="27")\r
358         {\r
359                 StartRedirectRequest(message);\r
360                 return true;\r
361         }\r
362 \r
363         // if this is a fatal error - insert index into database so we won't try to download this index again\r
364         if(message["Fatal"]=="true")\r
365         {\r
366                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
367                 st.Bind(0,identityid);\r
368                 st.Bind(1,idparts[4]);\r
369                 st.Bind(2,index);\r
370                 st.Step();\r
371                 st.Finalize();\r
372 \r
373                 m_log->error(m_fcpuniquename+"::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);\r
374         }\r
375 \r
376         // remove this identityid from request list\r
377         RemoveFromRequestList(identityid);\r
378 \r
379         return true;\r
380 }\r
381 \r
382 void MessageListRequester::Initialize()\r
383 {\r
384         m_fcpuniquename="ActiveMessageListRequester";\r
385         std::string tempval("");\r
386         m_maxrequests=0;\r
387         Option option(m_db);\r
388 \r
389         option.GetInt("MaxMessageListRequests",m_maxrequests);\r
390 \r
391         // active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2\r
392         m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);\r
393 \r
394         if(m_maxrequests<1)\r
395         {\r
396                 m_maxrequests=1;\r
397                 m_log->error("Option MaxMessageListRequests is currently set at "+tempval+".  It must be 1 or greater.");\r
398         }\r
399         if(m_maxrequests>100)\r
400         {\r
401                 m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+".  This value might be incorrectly configured.");\r
402         }\r
403 \r
404         tempval="";\r
405         option.Get("LocalTrustOverridesPeerTrust",tempval);\r
406         if(tempval=="true")\r
407         {\r
408                 m_localtrustoverrides=true;\r
409         }\r
410         else\r
411         {\r
412                 m_localtrustoverrides=false;\r
413         }\r
414 \r
415         tempval="";\r
416         option.Get("SaveMessagesFromNewBoards",tempval);\r
417         if(tempval=="true")\r
418         {\r
419                 m_savetonewboards=true;\r
420         }\r
421         else\r
422         {\r
423                 m_savetonewboards=false;\r
424         }\r
425 \r
426         m_messagedownloadmaxdaysbackward=5;\r
427         tempval="5";\r
428         option.Get("MessageDownloadMaxDaysBackward",tempval);\r
429         StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
430 \r
431         m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0);\r
432 \r
433 }\r
434 \r
435 void MessageListRequester::PopulateIDList()\r
436 {\r
437         Poco::DateTime date;\r
438         Poco::DateTime yesterday=date-Poco::Timespan(1,0,0,0,0);\r
439         int id;\r
440 \r
441         SQLite3DB::Statement st;\r
442 \r
443         // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
444         if(m_localtrustoverrides==false)\r
445         {\r
446                 st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
447         }\r
448         else\r
449         {\r
450                 st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
451         }\r
452         st.Step();\r
453 \r
454         m_ids.clear();\r
455 \r
456         while(st.RowReturned())\r
457         {\r
458                 st.ResultInt(0,id);\r
459                 m_ids[id]=false;\r
460                 st.Step();\r
461         }\r
462 }\r
463 \r
464 void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)\r
465 {\r
466         std::vector<std::string> parts;\r
467         std::string indexstr="";\r
468         std::string identityidstr="";\r
469         std::string datestr="";\r
470         FCPv2::Message newmessage;\r
471 \r
472         // get the new edition #\r
473         StringFunctions::Split(message["RedirectURI"],"/",parts);\r
474         //edition # is 2nd to last part\r
475         if(parts.size()>2)\r
476         {\r
477                 indexstr=parts[parts.size()-2];\r
478         }\r
479 \r
480         // get identityid\r
481         parts.clear();\r
482         StringFunctions::Split(message["Identifier"],"|",parts);\r
483         if(parts.size()>1)\r
484         {\r
485                 identityidstr=parts[1];\r
486         }\r
487         if(parts.size()>4)\r
488         {\r
489                 datestr=parts[4];\r
490         }\r
491 \r
492         newmessage.SetName("ClientGet");\r
493         newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);\r
494         newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];\r
495         newmessage["ReturnType"]="direct";\r
496         newmessage["MaxSize"]="1000000";\r
497 \r
498         m_fcp->Send(newmessage);\r
499 \r
500 }\r
501 \r
502 void MessageListRequester::StartRequest(const long &id)\r
503 {\r
504         Poco::DateTime now;\r
505         FCPv2::Message message;\r
506         std::string publickey;\r
507         int index=0;\r
508         std::string indexstr;\r
509         std::string identityidstr;\r
510 \r
511         SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
512         st.Bind(0,id);\r
513         st.Step();\r
514 \r
515         if(st.RowReturned())\r
516         {\r
517                 st.ResultText(0,publickey);\r
518 \r
519                 now=Poco::Timestamp();\r
520 \r
521                 SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");\r
522                 st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));\r
523                 st2.Bind(1,id);\r
524                 st2.Step();\r
525 \r
526                 index=0;\r
527                 if(st2.RowReturned())\r
528                 {\r
529                         if(st2.ResultNull(0)==false)\r
530                         {\r
531                                 st2.ResultInt(0,index);\r
532                                 // don't increment index here - the node will let us know if there is a new edition\r
533                                 // 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index\r
534                                 index++;\r
535                         }\r
536                 }\r
537                 st2.Finalize();\r
538 \r
539                 StringFunctions::Convert(index,indexstr);\r
540                 StringFunctions::Convert(id,identityidstr);\r
541 \r
542                 message.SetName("ClientGet");\r
543                 message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";\r
544                 message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];\r
545                 message["ReturnType"]="direct";\r
546                 message["MaxSize"]="1000000";\r
547 \r
548                 m_fcp->Send(message);\r
549 \r
550                 m_requesting.push_back(id);\r
551         }\r
552         st.Finalize();\r
553 \r
554         m_ids[id]=true;\r
555 }\r