src/freenet/messagerequester.cpp

   1 #include "../../include/freenet/messagerequester.h"\r
   2 #include "../../include/freenet/messagexml.h"\r
   3 \r
   4 #include <algorithm>\r
   5 \r
   6 #include <Poco/DateTime.h>\r
   7 #include <Poco/DateTimeFormatter.h>\r
   8 #include <Poco/Timespan.h>\r
   9 \r
  10 #ifdef XMEM\r
  11         #include <xmem.h>\r
  12 #endif\r
  13 \r
  14 MessageRequester::MessageRequester(SQLite3DB::DB *db):IIndexRequester<std::string>(db)\r
  15 {\r
  16         Initialize();\r
  17 }\r
  18 \r
  19 MessageRequester::MessageRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<std::string>(db,fcp)\r
  20 {\r
  21         Initialize();\r
  22 }\r
  23 \r
  24 const long MessageRequester::GetBoardID(const std::string &boardname, const std::string &identityname)\r
  25 {\r
  26         std::string lowerboard=boardname;\r
  27         StringFunctions::LowerCase(lowerboard,lowerboard);\r
  28         SQLite3DB::Statement st=m_db->Prepare("SELECT BoardID FROM tblBoard WHERE BoardName=?;");\r
  29         st.Bind(0,lowerboard);\r
  30         st.Step();\r
  31 \r
  32         if(st.RowReturned())\r
  33         {\r
  34                 int boardid;\r
  35                 st.ResultInt(0,boardid);\r
  36                 return boardid;\r
  37         }\r
  38         else\r
  39         {\r
  40                 Poco::DateTime now;\r
  41                 st=m_db->Prepare("INSERT INTO tblBoard(BoardName,DateAdded,SaveReceivedMessages,AddedMethod) VALUES(?,?,?,?);");\r
  42                 st.Bind(0,boardname);\r
  43                 st.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S"));\r
  44                 if(m_savemessagesfromnewboards)\r
  45                 {\r
  46                         st.Bind(2,"true");\r
  47                 }\r
  48                 else\r
  49                 {\r
  50                         st.Bind(2,"false");\r
  51                 }\r
  52                 st.Bind(3,"Message from "+identityname);\r
  53                 st.Step(true);\r
  54                 return st.GetLastInsertRowID();\r
  55         }       \r
  56 }\r
  57 \r
  58 const std::string MessageRequester::GetIdentityName(const long identityid)\r
  59 {\r
  60         SQLite3DB::Statement st=m_db->Prepare("SELECT Name,PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
  61         st.Bind(0,identityid);\r
  62         st.Step();\r
  63         if(st.RowReturned())\r
  64         {\r
  65                 std::vector<std::string> keyparts;\r
  66                 std::string key;\r
  67                 std::string name;\r
  68                 st.ResultText(0,name);\r
  69                 st.ResultText(1,key);\r
  70                 \r
  71                 StringFunctions::SplitMultiple(key,"@,",keyparts);\r
  72                 \r
  73                 if(keyparts.size()>1)\r
  74                 {\r
  75                         return name+"@"+keyparts[1];\r
  76                 }\r
  77                 else\r
  78                 {\r
  79                         return name+"@invalidpublickey";\r
  80                 }\r
  81         }\r
  82         else\r
  83         {\r
  84                 return "";\r
  85         }\r
  86 }\r
  87 \r
  88 const bool MessageRequester::HandleAllData(FCPv2::Message &message)\r
  89 {\r
  90         SQLite3DB::Statement st;\r
  91         std::vector<std::string> idparts;\r
  92         long datalength;\r
  93         std::vector<char> data;\r
  94         MessageXML xml;\r
  95         long identityid;\r
  96         long index;\r
  97         bool inserted=false;\r
  98         bool validmessage=true;\r
  99         long savetoboardcount=0;\r
 100 \r
 101         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 102         StringFunctions::Convert(message["DataLength"],datalength);\r
 103         StringFunctions::Convert(idparts[2],identityid);\r
 104         StringFunctions::Convert(idparts[4],index);\r
 105 \r
 106         // wait for all data to be received from connection\r
 107         m_fcp->WaitForBytes(1000,datalength);\r
 108 \r
 109         // if we got disconnected- return immediately\r
 110         if(m_fcp->IsConnected()==false)\r
 111         {\r
 112                 return false;\r
 113         }\r
 114 \r
 115         // receive the file\r
 116         m_fcp->Receive(data,datalength);\r
 117 \r
 118         // mark this index as received\r
 119         st=m_db->Prepare("UPDATE tblMessageRequests SET Found='true' WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
 120         st.Bind(0,identityid);\r
 121         st.Bind(1,idparts[3]);\r
 122         st.Bind(2,index);\r
 123         st.Step();\r
 124         st.Finalize();\r
 125 \r
 126         // parse file into xml and update the database\r
 127         if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
 128         {\r
 129                 std::vector<std::string> boards=xml.GetBoards();\r
 130                 std::map<long,std::string> replyto=xml.GetInReplyTo();\r
 131 \r
 132                 if(boards.size()>m_maxboardspermessage)\r
 133                 {\r
 134                         boards.resize(m_maxboardspermessage);\r
 135                 }\r
 136 \r
 137                 if(boards.size()<=0)\r
 138                 {\r
 139                         m_log->error("MessageRequester::HandleAllData Message XML did not contain any boards! "+message["Identifier"]);\r
 140                         // remove this identityid from request list\r
 141                         RemoveFromRequestList(idparts[1]);                      \r
 142                         return true;\r
 143                 }\r
 144                 if(xml.GetReplyBoard()=="")\r
 145                 {\r
 146                         m_log->error("MessageRequester::HandleAllData Message XML did not contain a reply board! "+message["Identifier"]);\r
 147                         // remove this identityid from request list\r
 148                         RemoveFromRequestList(idparts[1]);                      \r
 149                         return true;\r
 150                 }\r
 151 \r
 152                 // make sure the reply board is on the board list we are saving - if not, replace the last element of boards with the reply board\r
 153                 if(xml.GetReplyBoard()!="" && std::find(boards.begin(),boards.end(),xml.GetReplyBoard())==boards.end() && boards.size()>0)\r
 154                 {\r
 155                         boards[boards.size()-1]=xml.GetReplyBoard();\r
 156                 }\r
 157 \r
 158                 // make sure domain of message id match 43 characters of public key of identity (remove - and ~) - if not, discard message\r
 159                 // implement after 0.1.12 is released\r
 160                 st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
 161                 st.Bind(0,identityid);\r
 162                 st.Step();\r
 163                 if(st.RowReturned())\r
 164                 {\r
 165                         std::vector<std::string> uuidparts;\r
 166                         std::vector<std::string> keyparts;\r
 167                         std::string keypart="";\r
 168                         std::string publickey="";\r
 169 \r
 170                         st.ResultText(0,publickey);\r
 171 \r
 172                         StringFunctions::SplitMultiple(publickey,"@,",keyparts);\r
 173                         StringFunctions::SplitMultiple(xml.GetMessageID(),"@",uuidparts);\r
 174 \r
 175                         if(uuidparts.size()>1 && keyparts.size()>1)\r
 176                         {\r
 177                                 keypart=StringFunctions::Replace(StringFunctions::Replace(keyparts[1],"-",""),"~","");\r
 178                                 if(keypart!=uuidparts[1])\r
 179                                 {\r
 180                                         m_log->error("MessageRequester::HandleAllData MessageID in Message doesn't match public key of identity : "+message["Identifier"]);\r
 181                                         validmessage=false;\r
 182                                 }\r
 183                         }\r
 184                         else\r
 185                         {\r
 186                                 m_log->error("MessageRequester::HandleAllData Error with identity's public key or Message ID : "+message["Identifier"]);\r
 187                                 validmessage=false;\r
 188                         }\r
 189                 }\r
 190                 else\r
 191                 {\r
 192                         m_log->error("MessageRequester::HandleAllData Error couldn't find identity : "+message["Identifier"]);\r
 193                         validmessage=false;\r
 194                 }\r
 195 \r
 196                 // make sure we will at least save to 1 board before inserting message\r
 197                 savetoboardcount=0;\r
 198                 for(std::vector<std::string>::iterator bi=boards.begin(); bi!=boards.end(); bi++)\r
 199                 {\r
 200                         if(SaveToBoard((*bi)))\r
 201                         {\r
 202                                 savetoboardcount++;\r
 203                         }\r
 204                 }\r
 205 \r
 206                 if(validmessage && savetoboardcount>0)\r
 207                 {\r
 208                         std::string nntpbody="";\r
 209                         nntpbody=xml.GetBody();\r
 210 \r
 211                         //add file keys/sizes to body\r
 212                         std::vector<MessageXML::fileattachment> fileattachments=xml.GetFileAttachments();\r
 213                         if(fileattachments.size()>0)\r
 214                         {\r
 215                                 nntpbody+="\r\nAttachments";\r
 216                         }\r
 217                         for(std::vector<MessageXML::fileattachment>::iterator i=fileattachments.begin(); i!=fileattachments.end(); i++)\r
 218                         {\r
 219                                 std::string sizestr="0";\r
 220                                 StringFunctions::Convert((*i).m_size,sizestr);\r
 221 \r
 222                                 nntpbody+="\r\n"+(*i).m_key;\r
 223                                 nntpbody+="\r\n"+sizestr+" bytes";\r
 224                                 nntpbody+="\r\n";\r
 225                         }\r
 226 \r
 227                         m_db->Execute("BEGIN;");\r
 228 \r
 229                         st=m_db->Prepare("INSERT INTO tblMessage(IdentityID,FromName,MessageDate,MessageTime,Subject,MessageUUID,ReplyBoardID,Body,MessageIndex) VALUES(?,?,?,?,?,?,?,?,?);");\r
 230                         st.Bind(0,identityid);\r
 231                         st.Bind(1,GetIdentityName(identityid));\r
 232                         st.Bind(2,xml.GetDate());\r
 233                         st.Bind(3,xml.GetTime());\r
 234                         st.Bind(4,xml.GetSubject());\r
 235                         st.Bind(5,xml.GetMessageID());\r
 236                         st.Bind(6,GetBoardID(xml.GetReplyBoard(),GetIdentityName(identityid)));\r
 237                         st.Bind(7,nntpbody);\r
 238                         st.Bind(8,index);\r
 239                         inserted=st.Step(true);\r
 240                         int messageid=st.GetLastInsertRowID();\r
 241 \r
 242                         if(inserted==true)\r
 243                         {\r
 244 \r
 245                                 st=m_db->Prepare("INSERT INTO tblMessageBoard(MessageID,BoardID) VALUES(?,?);");\r
 246                                 for(std::vector<std::string>::iterator i=boards.begin(); i!=boards.end(); i++)\r
 247                                 {\r
 248                                         if(SaveToBoard((*i)))\r
 249                                         {\r
 250                                                 st.Bind(0,messageid);\r
 251                                                 st.Bind(1,GetBoardID((*i),GetIdentityName(identityid)));\r
 252                                                 st.Step();\r
 253                                                 st.Reset();\r
 254                                         }\r
 255                                 }\r
 256                                 st.Finalize();\r
 257 \r
 258                                 st=m_db->Prepare("INSERT INTO tblMessageReplyTo(MessageID,ReplyToMessageUUID,ReplyOrder) VALUES(?,?,?);");\r
 259                                 for(std::map<long,std::string>::iterator j=replyto.begin(); j!=replyto.end(); j++)\r
 260                                 {\r
 261                                         st.Bind(0,messageid);\r
 262                                         st.Bind(1,(*j).second);\r
 263                                         st.Bind(2,(*j).first);\r
 264                                         st.Step();\r
 265                                         st.Reset();\r
 266                                 }\r
 267                                 st.Finalize();\r
 268 \r
 269                                 m_log->debug("MessageRequester::HandleAllData parsed Message XML file : "+message["Identifier"]);\r
 270 \r
 271                         }\r
 272                         else    // couldn't insert - was already in database\r
 273                         {\r
 274                                 //m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageRequester::HandleAddData could not insert message into database.  "+message["Identifier"]);\r
 275                         }\r
 276 \r
 277                         st.Finalize();\r
 278 \r
 279                         m_db->Execute("COMMIT;");\r
 280 \r
 281                 }       // if validmessage\r
 282         }\r
 283         else\r
 284         {\r
 285                 m_log->error("MessageRequester::HandleAllData error parsing Message XML file : "+message["Identifier"]);\r
 286         }\r
 287 \r
 288         RemoveFromRequestList(idparts[1]);\r
 289 \r
 290         return true;\r
 291 }\r
 292 \r
 293 const bool MessageRequester::HandleGetFailed(FCPv2::Message &message)\r
 294 {\r
 295         SQLite3DB::Statement st;\r
 296         std::vector<std::string> idparts;\r
 297         std::string requestid;\r
 298         long index;\r
 299         long identityid;\r
 300 \r
 301         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 302         requestid=idparts[1];\r
 303         StringFunctions::Convert(idparts[2],identityid);\r
 304         StringFunctions::Convert(idparts[4],index);\r
 305 \r
 306         // if this is a fatal error - insert index into database so we won't try to download this index again\r
 307         if(message["Fatal"]=="true")\r
 308         {\r
 309                 st=m_db->Prepare("UPDATE tblMessageRequests SET Found='true' WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
 310                 st.Bind(0,identityid);\r
 311                 st.Bind(1,idparts[3]);\r
 312                 st.Bind(2,index);\r
 313                 st.Step();\r
 314                 st.Finalize();\r
 315 \r
 316                 m_log->error("MessageRequester::HandleGetFailed fatal error requesting "+message["Identifier"]);\r
 317         }\r
 318 \r
 319         // increase the failure count of the identity who gave us this index\r
 320         st=m_db->Prepare("UPDATE tblIdentity SET FailureCount=FailureCount+1 WHERE IdentityID IN (SELECT FromIdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?);");\r
 321         st.Bind(0,identityid);\r
 322         st.Bind(1,idparts[3]);\r
 323         st.Bind(2,index);\r
 324         st.Step();\r
 325         st.Finalize();\r
 326 \r
 327         // remove this identityid from request list\r
 328         RemoveFromRequestList(requestid);\r
 329 \r
 330         return true;\r
 331 }\r
 332 \r
 333 void MessageRequester::Initialize()\r
 334 {\r
 335         m_fcpuniquename="MessageRequester";\r
 336         std::string tempval("");\r
 337         m_maxrequests=0;\r
 338         Option option(m_db);\r
 339 \r
 340         option.GetInt("MaxMessageRequests",m_maxrequests);\r
 341         if(m_maxrequests<1)\r
 342         {\r
 343                 m_maxrequests=1;\r
 344                 m_log->error("Option MaxMessageRequests is currently set at "+tempval+".  It must be 1 or greater.");\r
 345         }\r
 346         if(m_maxrequests>100)\r
 347         {\r
 348                 m_log->warning("Option MaxMessageRequests is currently set at "+tempval+".  This value might be incorrectly configured.");\r
 349         }\r
 350 \r
 351         m_maxdaysbackward=0;\r
 352         option.GetInt("MessageDownloadMaxDaysBackward",m_maxdaysbackward);\r
 353         if(m_maxdaysbackward<0)\r
 354         {\r
 355                 m_maxdaysbackward=0;\r
 356                 m_log->error("Option MessageDownloadMaxDaysBackward is currently set at "+tempval+".  It must be 0 or greater.");\r
 357         }\r
 358         if(m_maxdaysbackward>30)\r
 359         {\r
 360                 m_log->warning("Option MessageDownloadMaxDaysBackward is currently set at "+tempval+".  This value might be incorrectly configured.");\r
 361         }\r
 362 \r
 363         m_maxpeermessages=0;\r
 364         option.GetInt("MaxPeerMessagesPerDay",m_maxpeermessages);\r
 365         if(m_maxpeermessages<1)\r
 366         {\r
 367                 m_maxpeermessages=1;\r
 368                 m_log->error("Option MaxPeerMessagesPerDay is currently set at "+tempval+".  It must be 1 or greater.");\r
 369         }\r
 370         if(m_maxpeermessages<20 || m_maxpeermessages>1000)\r
 371         {\r
 372                 m_log->warning("Option MaxPeerMessagesPerDay is currently set at "+tempval+".  This value might be incorrectly configured.  The suggested value is 200.");\r
 373         }\r
 374 \r
 375         m_maxboardspermessage=0;\r
 376         option.GetInt("MaxBoardsPerMessage",m_maxboardspermessage);\r
 377         if(m_maxboardspermessage<1)\r
 378         {\r
 379                 m_maxboardspermessage=1;\r
 380                 m_log->error("Option MaxBoardsPerMessage is currently set at "+tempval+".  It must be 1 or greater.");\r
 381         }\r
 382         if(m_maxboardspermessage>20)\r
 383         {\r
 384                 m_log->warning("Option MaxBoardsPerMessage is currently set at "+tempval+".  This value might be incorrectly configured.");\r
 385         }\r
 386 \r
 387         option.Get("SaveMessagesFromNewBoards",tempval);\r
 388         if(tempval=="true")\r
 389         {\r
 390                 m_savemessagesfromnewboards=true;\r
 391         }\r
 392         else\r
 393         {\r
 394                 m_savemessagesfromnewboards=false;\r
 395         }\r
 396 \r
 397         option.Get("LocalTrustOverridesPeerTrust",tempval);\r
 398         if(tempval=="true")\r
 399         {\r
 400                 m_localtrustoverrides=true;\r
 401         }\r
 402         else\r
 403         {\r
 404                 m_localtrustoverrides=false;\r
 405         }\r
 406 \r
 407 }\r
 408 \r
 409 void MessageRequester::PopulateIDList()\r
 410 {\r
 411         Poco::DateTime date;\r
 412         std::string val1;\r
 413         std::string val2;\r
 414         std::string val3;\r
 415         std::string sql;\r
 416         long requestindex;\r
 417 \r
 418         date-=Poco::Timespan(m_maxdaysbackward,0,0,0,0);\r
 419 \r
 420         sql="SELECT tblIdentity.IdentityID,Day,RequestIndex ";\r
 421         sql+="FROM tblMessageRequests INNER JOIN tblIdentity ON tblMessageRequests.IdentityID=tblIdentity.IdentityID ";\r
 422         sql+="WHERE FromMessageList='true' AND Found='false' AND Day>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' ";\r
 423         if(m_localtrustoverrides==false)\r
 424         {\r
 425                 sql+="AND (tblIdentity.LocalMessageTrust IS NULL OR tblIdentity.LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) ";\r
 426                 sql+="AND (tblIdentity.PeerMessageTrust IS NULL OR tblIdentity.PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ";\r
 427         }\r
 428         else\r
 429         {\r
 430                 sql+="AND (tblIdentity.LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (tblIdentity.LocalMessageTrust IS NULL AND (tblIdentity.PeerMessageTrust IS NULL OR tblIdentity.PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ";\r
 431         }\r
 432         sql+="AND tblIdentity.Name <> '' AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ";\r
 433         // sort by day descending - in case there is a bunch of messages on a day that keep timing out, we will eventually get to the next day and hopefully find messages there\r
 434         // secondary ascending sort on tries\r
 435         // tertiary sort on request index (so we get low indexes first)\r
 436         sql+="ORDER BY tblMessageRequests.Day DESC, tblMessageRequests.Tries ASC, tblMessageRequests.RequestIndex ASC ";\r
 437         sql+=";";\r
 438 \r
 439         SQLite3DB::Statement st=m_db->Prepare(sql);\r
 440         st.Step();\r
 441 \r
 442         m_ids.clear();\r
 443 \r
 444         while(st.RowReturned())\r
 445         {\r
 446                 st.ResultText(0,val1);\r
 447                 st.ResultText(1,val2);\r
 448                 st.ResultText(2,val3);\r
 449 \r
 450                 requestindex=0;\r
 451                 StringFunctions::Convert(val3,requestindex);\r
 452 \r
 453                 // only continue if index is < max messages we will accept from a peer\r
 454                 if(requestindex<m_maxpeermessages)\r
 455                 {\r
 456                         if(m_ids.find(val1+"*"+val2+"*"+val3)==m_ids.end())\r
 457                         {\r
 458                                 m_ids[val1+"*"+val2+"*"+val3]=false;\r
 459                         }\r
 460                 }\r
 461                 st.Step();\r
 462         }\r
 463 \r
 464 }\r
 465 \r
 466 const bool MessageRequester::SaveToBoard(const std::string &boardname)\r
 467 {\r
 468         bool save=true;\r
 469         SQLite3DB::Statement st=m_db->Prepare("SELECT SaveReceivedMessages FROM tblBoard WHERE BoardName=?;");\r
 470         st.Bind(0,boardname);\r
 471         st.Step();\r
 472         if(st.RowReturned())\r
 473         {\r
 474                 std::string val="";\r
 475                 st.ResultText(0,val);\r
 476                 if(val=="true")\r
 477                 {\r
 478                         save=true;\r
 479                 }\r
 480                 else\r
 481                 {\r
 482                         save=false;\r
 483                 }\r
 484         }\r
 485         return save;\r
 486 }\r
 487 \r
 488 void MessageRequester::StartRequest(const std::string &requestid)\r
 489 {\r
 490         FCPv2::Message message;\r
 491         std::vector<std::string> parts;\r
 492         std::string tempval;\r
 493         long identityid;\r
 494         std::string date;\r
 495         std::string indexstr;\r
 496         std::string publickey;\r
 497 \r
 498         StringFunctions::Split(requestid,"*",parts);\r
 499         StringFunctions::Convert(parts[0],identityid);\r
 500         StringFunctions::Convert(parts[1],date);\r
 501         indexstr=parts[2];\r
 502 \r
 503         SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
 504         st.Bind(0,identityid);\r
 505         st.Step();\r
 506 \r
 507         if(st.RowReturned())\r
 508         {\r
 509                 st.ResultText(0,publickey);\r
 510 \r
 511                 message.SetName("ClientGet");\r
 512                 message["URI"]=publickey+m_messagebase+"|"+date+"|Message|"+indexstr+".xml";\r
 513                 message["Identifier"]=m_fcpuniquename+"|"+requestid+"|"+parts[0]+"|"+parts[1]+"|"+parts[2]+"|"+message["URI"];\r
 514                 message["ReturnType"]="direct";\r
 515                 message["MaxSize"]="1000000";           // 1 MB\r
 516                 // don't use ULPR - we wan't to know of failures ASAP so we can mark them as such\r
 517                 //message["MaxRetries"]="-1";                   // use ULPR since we are fairly sure message exists since the author says it does\r
 518 \r
 519                 m_fcp->Send(message);\r
 520 \r
 521                 m_requesting.push_back(requestid);\r
 522 \r
 523                 // update tries\r
 524                 st=m_db->Prepare("UPDATE tblMessageRequests SET Tries=Tries+1 WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
 525                 st.Bind(0,identityid);\r
 526                 st.Bind(1,date);\r
 527                 st.Bind(2,indexstr);\r
 528                 st.Step();\r
 529 \r
 530                 m_log->debug("MessageRequester::StartRequest requesting "+message["Identifier"]);\r
 531         }\r
 532         \r
 533         m_ids[requestid]=true;\r
 534 \r
 535 }\r