version 0.3.32
[fms.git] / src / freenet / messagerequester.cpp
1 #include "../../include/freenet/messagerequester.h"\r
2 #include "../../include/freenet/messagexml.h"\r
3 \r
4 #include <algorithm>\r
5 \r
6 #include <Poco/DateTime.h>\r
7 #include <Poco/DateTimeFormatter.h>\r
8 #include <Poco/Timespan.h>\r
9 \r
10 #ifdef XMEM\r
11         #include <xmem.h>\r
12 #endif\r
13 \r
14 std::string MessageRequester::m_validuuidchars="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890~@_-";\r
15 \r
16 MessageRequester::MessageRequester(SQLite3DB::DB *db):IIndexRequester<std::string>(db)\r
17 {\r
18         Initialize();\r
19 }\r
20 \r
21 MessageRequester::MessageRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<std::string>(db,fcp)\r
22 {\r
23         Initialize();\r
24 }\r
25 \r
26 const long MessageRequester::GetBoardID(const std::string &boardname, const std::string &identityname)\r
27 {\r
28         std::string lowerboard=boardname;\r
29         StringFunctions::LowerCase(lowerboard,lowerboard);\r
30         SQLite3DB::Statement st=m_db->Prepare("SELECT BoardID FROM tblBoard WHERE BoardName=?;");\r
31         st.Bind(0,lowerboard);\r
32         st.Step();\r
33 \r
34         if(st.RowReturned())\r
35         {\r
36                 int boardid;\r
37                 st.ResultInt(0,boardid);\r
38                 return boardid;\r
39         }\r
40         else\r
41         {\r
42                 Poco::DateTime now;\r
43                 st=m_db->Prepare("INSERT INTO tblBoard(BoardName,DateAdded,SaveReceivedMessages,AddedMethod) VALUES(?,?,?,?);");\r
44                 st.Bind(0,boardname);\r
45                 st.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S"));\r
46                 if(m_savemessagesfromnewboards)\r
47                 {\r
48                         st.Bind(2,"true");\r
49                 }\r
50                 else\r
51                 {\r
52                         st.Bind(2,"false");\r
53                 }\r
54                 st.Bind(3,"Message from "+identityname);\r
55                 st.Step(true);\r
56                 return st.GetLastInsertRowID();\r
57         }       \r
58 }\r
59 \r
60 const std::string MessageRequester::GetIdentityName(const long identityid)\r
61 {\r
62         SQLite3DB::Statement st=m_db->Prepare("SELECT Name,PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
63         st.Bind(0,identityid);\r
64         st.Step();\r
65         if(st.RowReturned())\r
66         {\r
67                 std::vector<std::string> keyparts;\r
68                 std::string key;\r
69                 std::string name;\r
70                 st.ResultText(0,name);\r
71                 st.ResultText(1,key);\r
72                 \r
73                 StringFunctions::SplitMultiple(key,"@,",keyparts);\r
74                 \r
75                 if(keyparts.size()>1)\r
76                 {\r
77                         return name+"@"+keyparts[1];\r
78                 }\r
79                 else\r
80                 {\r
81                         return name+"@invalidpublickey";\r
82                 }\r
83         }\r
84         else\r
85         {\r
86                 return "";\r
87         }\r
88 }\r
89 \r
90 const bool MessageRequester::HandleAllData(FCPv2::Message &message)\r
91 {\r
92         SQLite3DB::Statement st;\r
93         std::vector<std::string> idparts;\r
94         long datalength;\r
95         std::vector<char> data;\r
96         MessageXML xml;\r
97         long identityid;\r
98         long index;\r
99         bool inserted=false;\r
100         bool validmessage=true;\r
101         long savetoboardcount=0;\r
102 \r
103         StringFunctions::Split(message["Identifier"],"|",idparts);\r
104         StringFunctions::Convert(message["DataLength"],datalength);\r
105         StringFunctions::Convert(idparts[2],identityid);\r
106         StringFunctions::Convert(idparts[4],index);\r
107 \r
108         // wait for all data to be received from connection\r
109         m_fcp->WaitForBytes(1000,datalength);\r
110 \r
111         // if we got disconnected- return immediately\r
112         if(m_fcp->IsConnected()==false)\r
113         {\r
114                 return false;\r
115         }\r
116 \r
117         // receive the file\r
118         m_fcp->Receive(data,datalength);\r
119 \r
120         // mark this index as received\r
121         st=m_db->Prepare("UPDATE tblMessageRequests SET Found='true' WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
122         st.Bind(0,identityid);\r
123         st.Bind(1,idparts[3]);\r
124         st.Bind(2,index);\r
125         st.Step();\r
126         st.Finalize();\r
127 \r
128         // parse file into xml and update the database\r
129         if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
130         {\r
131                 std::vector<std::string> boards=xml.GetBoards();\r
132                 std::map<long,std::string> replyto=xml.GetInReplyTo();\r
133 \r
134                 if(boards.size()>m_maxboardspermessage)\r
135                 {\r
136                         boards.resize(m_maxboardspermessage);\r
137                 }\r
138 \r
139                 if(boards.size()<=0)\r
140                 {\r
141                         m_log->error("MessageRequester::HandleAllData Message XML did not contain any boards! "+message["Identifier"]);\r
142                         // remove this identityid from request list\r
143                         RemoveFromRequestList(idparts[1]);                      \r
144                         return true;\r
145                 }\r
146                 if(xml.GetReplyBoard()=="")\r
147                 {\r
148                         m_log->error("MessageRequester::HandleAllData Message XML did not contain a reply board! "+message["Identifier"]);\r
149                         // remove this identityid from request list\r
150                         RemoveFromRequestList(idparts[1]);                      \r
151                         return true;\r
152                 }\r
153 \r
154                 // make sure the reply board is on the board list we are saving - if not, replace the last element of boards with the reply board\r
155                 if(xml.GetReplyBoard()!="" && std::find(boards.begin(),boards.end(),xml.GetReplyBoard())==boards.end() && boards.size()>0)\r
156                 {\r
157                         boards[boards.size()-1]=xml.GetReplyBoard();\r
158                 }\r
159 \r
160                 // make sure domain of message id match 43 characters of public key of identity (remove - and ~) - if not, discard message\r
161                 // implement after 0.1.12 is released\r
162                 st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
163                 st.Bind(0,identityid);\r
164                 st.Step();\r
165                 if(st.RowReturned())\r
166                 {\r
167                         std::vector<std::string> uuidparts;\r
168                         std::vector<std::string> keyparts;\r
169                         std::string keypart="";\r
170                         std::string publickey="";\r
171 \r
172                         st.ResultText(0,publickey);\r
173 \r
174                         StringFunctions::SplitMultiple(publickey,"@,",keyparts);\r
175                         StringFunctions::SplitMultiple(xml.GetMessageID(),"@",uuidparts);\r
176 \r
177                         if(uuidparts.size()>1 && keyparts.size()>1 && xml.GetMessageID().find_first_not_of(m_validuuidchars)==std::string::npos)\r
178                         {\r
179                                 keypart=StringFunctions::Replace(StringFunctions::Replace(keyparts[1],"-",""),"~","");\r
180                                 if(keypart!=uuidparts[1])\r
181                                 {\r
182                                         m_log->error("MessageRequester::HandleAllData MessageID in Message doesn't match public key of identity : "+message["Identifier"]);\r
183                                         validmessage=false;\r
184                                 }\r
185                         }\r
186                         else\r
187                         {\r
188                                 m_log->error("MessageRequester::HandleAllData Error with identity's public key or Message ID : "+message["Identifier"]);\r
189                                 validmessage=false;\r
190                         }\r
191                 }\r
192                 else\r
193                 {\r
194                         m_log->error("MessageRequester::HandleAllData Error couldn't find identity : "+message["Identifier"]);\r
195                         validmessage=false;\r
196                 }\r
197 \r
198                 // make sure we will at least save to 1 board before inserting message\r
199                 savetoboardcount=0;\r
200                 for(std::vector<std::string>::iterator bi=boards.begin(); bi!=boards.end(); bi++)\r
201                 {\r
202                         if(SaveToBoard((*bi)))\r
203                         {\r
204                                 savetoboardcount++;\r
205                         }\r
206                 }\r
207 \r
208                 if(validmessage && savetoboardcount>0)\r
209                 {\r
210                         std::string nntpbody="";\r
211                         nntpbody=xml.GetBody();\r
212 \r
213                         //add file keys/sizes to body\r
214                         std::vector<MessageXML::fileattachment> fileattachments=xml.GetFileAttachments();\r
215                         if(fileattachments.size()>0)\r
216                         {\r
217                                 nntpbody+="\r\nAttachments";\r
218                         }\r
219                         for(std::vector<MessageXML::fileattachment>::iterator i=fileattachments.begin(); i!=fileattachments.end(); i++)\r
220                         {\r
221                                 std::string sizestr="0";\r
222                                 StringFunctions::Convert((*i).m_size,sizestr);\r
223 \r
224                                 nntpbody+="\r\n"+(*i).m_key;\r
225                                 nntpbody+="\r\n"+sizestr+" bytes";\r
226                                 nntpbody+="\r\n";\r
227                         }\r
228 \r
229                         m_db->Execute("BEGIN;");\r
230 \r
231                         st=m_db->Prepare("INSERT INTO tblMessage(IdentityID,FromName,MessageDate,MessageTime,Subject,MessageUUID,ReplyBoardID,Body,MessageIndex,InsertDate) VALUES(?,?,?,?,?,?,?,?,?,?);");\r
232                         st.Bind(0,identityid);\r
233                         st.Bind(1,GetIdentityName(identityid));\r
234                         st.Bind(2,xml.GetDate());\r
235                         st.Bind(3,xml.GetTime());\r
236                         st.Bind(4,xml.GetSubject());\r
237                         st.Bind(5,xml.GetMessageID());\r
238                         st.Bind(6,GetBoardID(xml.GetReplyBoard(),GetIdentityName(identityid)));\r
239                         st.Bind(7,nntpbody);\r
240                         st.Bind(8,index);\r
241                         st.Bind(9,idparts[3]);\r
242                         inserted=st.Step(true);\r
243                         long messageid=st.GetLastInsertRowID();\r
244 \r
245                         if(inserted==true)\r
246                         {\r
247 \r
248                                 st=m_db->Prepare("INSERT INTO tblMessageBoard(MessageID,BoardID) VALUES(?,?);");\r
249                                 for(std::vector<std::string>::iterator i=boards.begin(); i!=boards.end(); i++)\r
250                                 {\r
251                                         if(SaveToBoard((*i)))\r
252                                         {\r
253                                                 st.Bind(0,messageid);\r
254                                                 st.Bind(1,GetBoardID((*i),GetIdentityName(identityid)));\r
255                                                 st.Step();\r
256                                                 st.Reset();\r
257                                         }\r
258                                 }\r
259                                 st.Finalize();\r
260 \r
261                                 st=m_db->Prepare("INSERT INTO tblMessageReplyTo(MessageID,ReplyToMessageUUID,ReplyOrder) VALUES(?,?,?);");\r
262                                 for(std::map<long,std::string>::iterator j=replyto.begin(); j!=replyto.end(); j++)\r
263                                 {\r
264                                         st.Bind(0,messageid);\r
265                                         st.Bind(1,(*j).second);\r
266                                         st.Bind(2,(*j).first);\r
267                                         st.Step();\r
268                                         st.Reset();\r
269                                 }\r
270                                 st.Finalize();\r
271 \r
272                                 m_log->debug("MessageRequester::HandleAllData parsed Message XML file : "+message["Identifier"]);\r
273 \r
274                         }\r
275                         else    // couldn't insert - was already in database\r
276                         {\r
277                                 //m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageRequester::HandleAddData could not insert message into database.  "+message["Identifier"]);\r
278                         }\r
279 \r
280                         st.Finalize();\r
281 \r
282                         m_db->Execute("COMMIT;");\r
283 \r
284                 }       // if validmessage\r
285         }\r
286         else\r
287         {\r
288                 m_log->error("MessageRequester::HandleAllData error parsing Message XML file : "+message["Identifier"]);\r
289         }\r
290 \r
291         RemoveFromRequestList(idparts[1]);\r
292 \r
293         return true;\r
294 }\r
295 \r
296 const bool MessageRequester::HandleGetFailed(FCPv2::Message &message)\r
297 {\r
298         SQLite3DB::Statement st;\r
299         std::vector<std::string> idparts;\r
300         std::string requestid;\r
301         long index;\r
302         long identityid;\r
303 \r
304         StringFunctions::Split(message["Identifier"],"|",idparts);\r
305         requestid=idparts[1];\r
306         StringFunctions::Convert(idparts[2],identityid);\r
307         StringFunctions::Convert(idparts[4],index);\r
308 \r
309         // if this is a fatal error - insert index into database so we won't try to download this index again\r
310         if(message["Fatal"]=="true")\r
311         {\r
312                 st=m_db->Prepare("UPDATE tblMessageRequests SET Found='true' WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
313                 st.Bind(0,identityid);\r
314                 st.Bind(1,idparts[3]);\r
315                 st.Bind(2,index);\r
316                 st.Step();\r
317                 st.Finalize();\r
318 \r
319                 m_log->error("MessageRequester::HandleGetFailed fatal error requesting "+message["Identifier"]);\r
320         }\r
321 \r
322         // increase the failure count of the identity who gave us this index\r
323         st=m_db->Prepare("UPDATE tblIdentity SET FailureCount=FailureCount+1 WHERE IdentityID IN (SELECT FromIdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?);");\r
324         st.Bind(0,identityid);\r
325         st.Bind(1,idparts[3]);\r
326         st.Bind(2,index);\r
327         st.Step();\r
328         st.Finalize();\r
329 \r
330         // remove this identityid from request list\r
331         RemoveFromRequestList(requestid);\r
332 \r
333         return true;\r
334 }\r
335 \r
336 void MessageRequester::Initialize()\r
337 {\r
338         m_fcpuniquename="MessageRequester";\r
339         std::string tempval("");\r
340         m_maxrequests=0;\r
341         Option option(m_db);\r
342 \r
343         option.GetInt("MaxMessageRequests",m_maxrequests);\r
344         if(m_maxrequests<1)\r
345         {\r
346                 m_maxrequests=1;\r
347                 m_log->error("Option MaxMessageRequests is currently set at "+tempval+".  It must be 1 or greater.");\r
348         }\r
349         if(m_maxrequests>100)\r
350         {\r
351                 m_log->warning("Option MaxMessageRequests is currently set at "+tempval+".  This value might be incorrectly configured.");\r
352         }\r
353 \r
354         m_maxdaysbackward=0;\r
355         option.GetInt("MessageDownloadMaxDaysBackward",m_maxdaysbackward);\r
356         if(m_maxdaysbackward<0)\r
357         {\r
358                 m_maxdaysbackward=0;\r
359                 m_log->error("Option MessageDownloadMaxDaysBackward is currently set at "+tempval+".  It must be 0 or greater.");\r
360         }\r
361         if(m_maxdaysbackward>30)\r
362         {\r
363                 m_log->warning("Option MessageDownloadMaxDaysBackward is currently set at "+tempval+".  This value might be incorrectly configured.");\r
364         }\r
365 \r
366         m_maxpeermessages=0;\r
367         option.GetInt("MaxPeerMessagesPerDay",m_maxpeermessages);\r
368         if(m_maxpeermessages<1)\r
369         {\r
370                 m_maxpeermessages=1;\r
371                 m_log->error("Option MaxPeerMessagesPerDay is currently set at "+tempval+".  It must be 1 or greater.");\r
372         }\r
373         if(m_maxpeermessages<20 || m_maxpeermessages>1000)\r
374         {\r
375                 m_log->warning("Option MaxPeerMessagesPerDay is currently set at "+tempval+".  This value might be incorrectly configured.  The suggested value is 200.");\r
376         }\r
377 \r
378         m_maxboardspermessage=0;\r
379         option.GetInt("MaxBoardsPerMessage",m_maxboardspermessage);\r
380         if(m_maxboardspermessage<1)\r
381         {\r
382                 m_maxboardspermessage=1;\r
383                 m_log->error("Option MaxBoardsPerMessage is currently set at "+tempval+".  It must be 1 or greater.");\r
384         }\r
385         if(m_maxboardspermessage>20)\r
386         {\r
387                 m_log->warning("Option MaxBoardsPerMessage is currently set at "+tempval+".  This value might be incorrectly configured.");\r
388         }\r
389 \r
390         option.Get("SaveMessagesFromNewBoards",tempval);\r
391         if(tempval=="true")\r
392         {\r
393                 m_savemessagesfromnewboards=true;\r
394         }\r
395         else\r
396         {\r
397                 m_savemessagesfromnewboards=false;\r
398         }\r
399 \r
400         option.Get("LocalTrustOverridesPeerTrust",tempval);\r
401         if(tempval=="true")\r
402         {\r
403                 m_localtrustoverrides=true;\r
404         }\r
405         else\r
406         {\r
407                 m_localtrustoverrides=false;\r
408         }\r
409 \r
410 }\r
411 \r
412 void MessageRequester::PopulateIDList()\r
413 {\r
414         Poco::DateTime date;\r
415         std::string val1;\r
416         std::string val2;\r
417         std::string val3;\r
418         std::string sql;\r
419         long requestindex;\r
420 \r
421         date-=Poco::Timespan(m_maxdaysbackward,0,0,0,0);\r
422 \r
423         sql="SELECT tblIdentity.IdentityID,Day,RequestIndex ";\r
424         sql+="FROM tblMessageRequests INNER JOIN tblIdentity ON tblMessageRequests.IdentityID=tblIdentity.IdentityID ";\r
425         sql+="WHERE FromMessageList='true' AND Found='false' AND Day>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' ";\r
426         if(m_localtrustoverrides==false)\r
427         {\r
428                 sql+="AND (tblIdentity.LocalMessageTrust IS NULL OR tblIdentity.LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) ";\r
429                 sql+="AND (tblIdentity.PeerMessageTrust IS NULL OR tblIdentity.PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ";\r
430         }\r
431         else\r
432         {\r
433                 sql+="AND (tblIdentity.LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (tblIdentity.LocalMessageTrust IS NULL AND (tblIdentity.PeerMessageTrust IS NULL OR tblIdentity.PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ";\r
434         }\r
435         sql+="AND tblIdentity.Name <> '' AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ";\r
436         // sort by day descending - in case there is a bunch of messages on a day that keep timing out, we will eventually get to the next day and hopefully find messages there\r
437         // secondary ascending sort on tries\r
438         // tertiary sort on request index (so we get low indexes first)\r
439         sql+="ORDER BY tblMessageRequests.Day DESC, tblMessageRequests.Tries ASC, tblMessageRequests.RequestIndex ASC ";\r
440         sql+=";";\r
441 \r
442         SQLite3DB::Statement st=m_db->Prepare(sql);\r
443         st.Step();\r
444 \r
445         m_ids.clear();\r
446 \r
447         while(st.RowReturned())\r
448         {\r
449                 st.ResultText(0,val1);\r
450                 st.ResultText(1,val2);\r
451                 st.ResultText(2,val3);\r
452 \r
453                 requestindex=0;\r
454                 StringFunctions::Convert(val3,requestindex);\r
455 \r
456                 // only continue if index is < max messages we will accept from a peer\r
457                 if(requestindex<m_maxpeermessages)\r
458                 {\r
459                         if(m_ids.find(val1+"*"+val2+"*"+val3)==m_ids.end())\r
460                         {\r
461                                 m_ids[val1+"*"+val2+"*"+val3]=false;\r
462                         }\r
463                 }\r
464                 st.Step();\r
465         }\r
466 \r
467 }\r
468 \r
469 const bool MessageRequester::SaveToBoard(const std::string &boardname)\r
470 {\r
471         bool save=true;\r
472         SQLite3DB::Statement st=m_db->Prepare("SELECT SaveReceivedMessages FROM tblBoard WHERE BoardName=?;");\r
473         st.Bind(0,boardname);\r
474         st.Step();\r
475         if(st.RowReturned())\r
476         {\r
477                 std::string val="";\r
478                 st.ResultText(0,val);\r
479                 if(val=="true")\r
480                 {\r
481                         save=true;\r
482                 }\r
483                 else\r
484                 {\r
485                         save=false;\r
486                 }\r
487         }\r
488         return save;\r
489 }\r
490 \r
491 void MessageRequester::StartRequest(const std::string &requestid)\r
492 {\r
493         FCPv2::Message message;\r
494         std::vector<std::string> parts;\r
495         std::string tempval;\r
496         long identityid;\r
497         std::string date;\r
498         std::string indexstr;\r
499         std::string publickey;\r
500 \r
501         StringFunctions::Split(requestid,"*",parts);\r
502         StringFunctions::Convert(parts[0],identityid);\r
503         StringFunctions::Convert(parts[1],date);\r
504         indexstr=parts[2];\r
505 \r
506         SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
507         st.Bind(0,identityid);\r
508         st.Step();\r
509 \r
510         if(st.RowReturned())\r
511         {\r
512                 st.ResultText(0,publickey);\r
513 \r
514                 message.SetName("ClientGet");\r
515                 message["URI"]=publickey+m_messagebase+"|"+date+"|Message|"+indexstr+".xml";\r
516                 message["Identifier"]=m_fcpuniquename+"|"+requestid+"|"+parts[0]+"|"+parts[1]+"|"+parts[2]+"|"+message["URI"];\r
517                 message["ReturnType"]="direct";\r
518                 message["MaxSize"]="1000000";           // 1 MB\r
519                 // don't use ULPR - we wan't to know of failures ASAP so we can mark them as such\r
520                 //message["MaxRetries"]="-1";                   // use ULPR since we are fairly sure message exists since the author says it does\r
521 \r
522                 m_fcp->Send(message);\r
523 \r
524                 m_requesting.push_back(requestid);\r
525 \r
526                 // update tries\r
527                 st=m_db->Prepare("UPDATE tblMessageRequests SET Tries=Tries+1 WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
528                 st.Bind(0,identityid);\r
529                 st.Bind(1,date);\r
530                 st.Bind(2,indexstr);\r
531                 st.Step();\r
532 \r
533                 m_log->debug("MessageRequester::StartRequest requesting "+message["Identifier"]);\r
534         }\r
535         \r
536         m_ids[requestid]=true;\r
537 \r
538 }\r