src/freenet/messagelistrequester.cpp

   1 #include "../../include/freenet/messagelistrequester.h"\r
   2 #include "../../include/freenet/messagelistxml.h"\r
   3 \r
   4 #include <Poco/DateTime.h>\r
   5 #include <Poco/DateTimeFormatter.h>\r
   6 #include <Poco/DateTimeParser.h>\r
   7 #include <Poco/Timestamp.h>\r
   8 \r
   9 #ifdef XMEM\r
  10         #include <xmem.h>\r
  11 #endif\r
  12 \r
  13 MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)\r
  14 {\r
  15         Initialize();\r
  16 }\r
  17 \r
  18 MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)\r
  19 {\r
  20         Initialize();\r
  21 }\r
  22 \r
  23 const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) const\r
  24 {\r
  25         std::vector<std::string> dateparts;\r
  26         int year=0;\r
  27         int month=0;\r
  28         int day=0;\r
  29         Poco::DateTime today;\r
  30 \r
  31         StringFunctions::Split(datestr,"-",dateparts);\r
  32         if(dateparts.size()==3)\r
  33         {\r
  34                 StringFunctions::Convert(dateparts[0],year);\r
  35                 StringFunctions::Convert(dateparts[1],month);\r
  36                 StringFunctions::Convert(dateparts[2],day);\r
  37                 if(today.year()>year || (today.year()==year && today.month()>month) || (today.year()==year && today.month()==month && today.day()>=day))\r
  38                 {\r
  39                         return true;\r
  40                 }\r
  41                 else\r
  42                 {\r
  43                         return false;\r
  44                 }\r
  45         }\r
  46         else\r
  47         {\r
  48                 return false;\r
  49         }\r
  50 \r
  51 }\r
  52 \r
  53 const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &datestr) const\r
  54 {\r
  55         Poco::DateTime checkdate;\r
  56         Poco::DateTime date;\r
  57         int tzdiff=0;\r
  58         if(Poco::DateTimeParser::tryParse(datestr,date,tzdiff))\r
  59         {\r
  60                 checkdate-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);\r
  61                 if(checkdate<=date)\r
  62                 {\r
  63                         return true;\r
  64                 }\r
  65                 else\r
  66                 {\r
  67                         return false;\r
  68                 }\r
  69         }\r
  70         else\r
  71         {\r
  72                 return false;\r
  73         }\r
  74 }\r
  75 \r
  76 void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)\r
  77 {\r
  78         SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
  79         st.Step();\r
  80         while(st.RowReturned())\r
  81         {\r
  82                 std::string boardname="";\r
  83                 std::string tempval="";\r
  84                 st.ResultText(0,boardname);\r
  85                 st.ResultText(1,tempval);\r
  86 \r
  87                 if(tempval=="true")\r
  88                 {\r
  89                         boards[boardname]=true;\r
  90                 }\r
  91                 else\r
  92                 {\r
  93                         boards[boardname]=false;\r
  94                 }\r
  95 \r
  96                 st.Step();\r
  97         }\r
  98 }\r
  99 \r
 100 const bool MessageListRequester::HandleAllData(FCPv2::Message &message)\r
 101 {       \r
 102         SQLite3DB::Statement st;\r
 103         SQLite3DB::Statement trustst;\r
 104         std::vector<std::string> idparts;\r
 105         long datalength;\r
 106         std::vector<char> data;\r
 107         MessageListXML xml;\r
 108         long identityid;\r
 109         long index;\r
 110         std::map<std::string,bool> boards;      // list of boards and if we will save messages for that board or not\r
 111         bool addmessage=false;\r
 112         std::string boardsstr="";\r
 113         std::string datestr="";\r
 114         std::vector<std::string> dateparts;\r
 115 \r
 116         GetBoardList(boards);\r
 117 \r
 118         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 119         StringFunctions::Convert(message["DataLength"],datalength);\r
 120         StringFunctions::Convert(idparts[1],identityid);\r
 121         StringFunctions::Convert(idparts[2],index);\r
 122 \r
 123         // wait for all data to be received from connection\r
 124         m_fcp->WaitForBytes(1000,datalength);\r
 125 \r
 126         // if we got disconnected- return immediately\r
 127         if(m_fcp->IsConnected()==false)\r
 128         {\r
 129                 return false;\r
 130         }\r
 131 \r
 132         // receive the file\r
 133         m_fcp->Receive(data,datalength);\r
 134 \r
 135         // parse file into xml and update the database\r
 136         if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
 137         {\r
 138 \r
 139                 m_db->Execute("BEGIN;");\r
 140 \r
 141                 SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
 142                 SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
 143                 SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");\r
 144                 for(long i=0; i<xml.MessageCount(); i++)\r
 145                 {\r
 146 \r
 147                         // go through each board the message was posted to and see if we are saving messages to that board\r
 148                         // if the board isn't found, see if we are saving messages to new boards\r
 149                         boardsstr="";\r
 150                         addmessage=false;\r
 151                         std::vector<std::string> messageboards=xml.GetBoards(i);\r
 152                         for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
 153                         {\r
 154                                 if(boards.find((*j))!=boards.end())\r
 155                                 {\r
 156                                         if(boards[(*j)]==true)\r
 157                                         {\r
 158                                                 addmessage=true;\r
 159                                         }\r
 160                                 }\r
 161                                 else if(m_savetonewboards==true)\r
 162                                 {\r
 163                                         addmessage=true;\r
 164                                 }\r
 165                                 if(j!=messageboards.begin())\r
 166                                 {\r
 167                                         boardsstr+=", ";\r
 168                                 }\r
 169                                 boardsstr+=(*j);\r
 170                         }\r
 171 \r
 172                         if(CheckDateNotFuture(xml.GetDate(i))==false)\r
 173                         {\r
 174                                 addmessage=false;\r
 175                                 m_log->error(m_fcpuniquename+"::HandleAllData date for message is in future! "+xml.GetDate(i));\r
 176                         }\r
 177 \r
 178                         if(addmessage==true && CheckDateWithinMaxDays(xml.GetDate(i))==false)\r
 179                         {\r
 180                                 addmessage=false;\r
 181                         }\r
 182 \r
 183                         if(addmessage==true)\r
 184                         {\r
 185                                 st.Bind(0,identityid);\r
 186                                 st.Bind(1,xml.GetDate(i));\r
 187                                 st.Bind(2,xml.GetIndex(i));\r
 188                                 st.Step();\r
 189                                 if(st.RowReturned()==false)\r
 190                                 {\r
 191                                         mst.Bind(0,identityid);\r
 192                                         mst.Bind(1,xml.GetDate(i));\r
 193                                         mst.Bind(2,xml.GetIndex(i));\r
 194                                         mst.Step();\r
 195                                         mst.Reset();\r
 196                                 }\r
 197                                 st.Reset();\r
 198                         }\r
 199                         else\r
 200                         {\r
 201                                 //m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr+" on "+xml.GetDate(i));\r
 202                         }\r
 203                 }\r
 204 \r
 205                 // insert external message indexes\r
 206                 for(long i=0; i<xml.ExternalMessageCount(); i++)\r
 207                 {\r
 208                         if(xml.GetExternalType(i)=="Keyed")\r
 209                         {\r
 210                                 // go through each board the message was posted to and see if we are saving messages to that board\r
 211                                 // if the board isn't found, see if we are saving messages to new boards\r
 212                                 boardsstr="";\r
 213                                 addmessage=false;\r
 214                                 std::vector<std::string> messageboards=xml.GetExternalBoards(i);\r
 215                                 for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
 216                                 {\r
 217                                         if(boards.find((*j))!=boards.end())\r
 218                                         {\r
 219                                                 if(boards[(*j)]==true)\r
 220                                                 {\r
 221                                                         addmessage=true;\r
 222                                                 }\r
 223                                         }\r
 224                                         else if(m_savetonewboards==true)\r
 225                                         {\r
 226                                                 addmessage=true;\r
 227                                         }\r
 228                                         if(j!=messageboards.begin())\r
 229                                         {\r
 230                                                 boardsstr+=", ";\r
 231                                         }\r
 232                                         boardsstr+=(*j);\r
 233                                 }\r
 234 \r
 235                                 if(CheckDateNotFuture(xml.GetExternalDate(i))==false)\r
 236                                 {\r
 237                                         addmessage=false;\r
 238                                         m_log->error(m_fcpuniquename+"::HandleAllData date for external message is in future! "+xml.GetExternalDate(i));\r
 239                                 }\r
 240 \r
 241                                 if(addmessage==true && CheckDateWithinMaxDays(xml.GetExternalDate(i))==false)\r
 242                                 {\r
 243                                         addmessage=false;\r
 244                                 }\r
 245 \r
 246                                 if(addmessage==true)\r
 247                                 {\r
 248                                         spk.Bind(0,xml.GetExternalIdentity(i));\r
 249                                         spk.Step();\r
 250                                         if(spk.RowReturned())\r
 251                                         {\r
 252                                                 int thisidentityid=0;\r
 253                                                 spk.ResultInt(0,thisidentityid);\r
 254                                                 mst.Bind(0,thisidentityid);\r
 255                                                 mst.Bind(1,xml.GetExternalDate(i));\r
 256                                                 mst.Bind(2,xml.GetExternalIndex(i));\r
 257                                                 mst.Step();\r
 258                                                 mst.Reset();\r
 259                                         }\r
 260                                         spk.Reset();\r
 261                                 }\r
 262                                 else\r
 263                                 {\r
 264                                         //m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i) + " on " + xml.GetExternalDate(i));\r
 265                                 }\r
 266                         }\r
 267                 }\r
 268 \r
 269                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");\r
 270                 st.Bind(0,identityid);\r
 271                 st.Bind(1,idparts[4]);\r
 272                 st.Bind(2,index);\r
 273                 st.Step();\r
 274                 st.Finalize();\r
 275 \r
 276                 m_db->Execute("COMMIT;");\r
 277 \r
 278                 m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
 279         }\r
 280         else\r
 281         {\r
 282                 // bad data - mark index\r
 283                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
 284                 st.Bind(0,identityid);\r
 285                 st.Bind(1,idparts[4]);\r
 286                 st.Bind(2,index);\r
 287                 st.Step();\r
 288                 st.Finalize();\r
 289 \r
 290                 m_log->error(m_fcpuniquename+"::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);\r
 291         }\r
 292 \r
 293         // remove this identityid from request list\r
 294         RemoveFromRequestList(identityid);\r
 295 \r
 296         return true;\r
 297 \r
 298 }\r
 299 \r
 300 const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)\r
 301 {\r
 302         SQLite3DB::Statement st;\r
 303         std::vector<std::string> idparts;\r
 304         long identityid;\r
 305         long index;\r
 306 \r
 307         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 308         StringFunctions::Convert(idparts[1],identityid);\r
 309         StringFunctions::Convert(idparts[2],index);     \r
 310 \r
 311         // code 27 - permanent redirect\r
 312         if(message["Code"]=="27")\r
 313         {\r
 314                 StartRedirectRequest(message);\r
 315                 return true;\r
 316         }\r
 317 \r
 318         // if this is a fatal error - insert index into database so we won't try to download this index again\r
 319         if(message["Fatal"]=="true")\r
 320         {\r
 321                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
 322                 st.Bind(0,identityid);\r
 323                 st.Bind(1,idparts[4]);\r
 324                 st.Bind(2,index);\r
 325                 st.Step();\r
 326                 st.Finalize();\r
 327 \r
 328                 m_log->error(m_fcpuniquename+"::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);\r
 329         }\r
 330 \r
 331         // remove this identityid from request list\r
 332         RemoveFromRequestList(identityid);\r
 333 \r
 334         return true;\r
 335 }\r
 336 \r
 337 void MessageListRequester::Initialize()\r
 338 {\r
 339         m_fcpuniquename="ActiveMessageListRequester";\r
 340         std::string tempval("");\r
 341         m_maxrequests=0;\r
 342         Option option(m_db);\r
 343 \r
 344         option.GetInt("MaxMessageListRequests",m_maxrequests);\r
 345 \r
 346         // active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2\r
 347         m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);\r
 348 \r
 349         if(m_maxrequests<1)\r
 350         {\r
 351                 m_maxrequests=1;\r
 352                 m_log->error("Option MaxMessageListRequests is currently set at "+tempval+".  It must be 1 or greater.");\r
 353         }\r
 354         if(m_maxrequests>100)\r
 355         {\r
 356                 m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+".  This value might be incorrectly configured.");\r
 357         }\r
 358 \r
 359         tempval="";\r
 360         option.Get("LocalTrustOverridesPeerTrust",tempval);\r
 361         if(tempval=="true")\r
 362         {\r
 363                 m_localtrustoverrides=true;\r
 364         }\r
 365         else\r
 366         {\r
 367                 m_localtrustoverrides=false;\r
 368         }\r
 369 \r
 370         tempval="";\r
 371         option.Get("SaveMessagesFromNewBoards",tempval);\r
 372         if(tempval=="true")\r
 373         {\r
 374                 m_savetonewboards=true;\r
 375         }\r
 376         else\r
 377         {\r
 378                 m_savetonewboards=false;\r
 379         }\r
 380 \r
 381         m_messagedownloadmaxdaysbackward=5;\r
 382         tempval="5";\r
 383         option.Get("MessageDownloadMaxDaysBackward",tempval);\r
 384         StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
 385 \r
 386 }\r
 387 \r
 388 void MessageListRequester::PopulateIDList()\r
 389 {\r
 390         Poco::DateTime date;\r
 391         Poco::DateTime yesterday=date-Poco::Timespan(1,0,0,0,0);\r
 392         int id;\r
 393 \r
 394         SQLite3DB::Statement st;\r
 395 \r
 396         // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
 397         if(m_localtrustoverrides==false)\r
 398         {\r
 399                 st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
 400         }\r
 401         else\r
 402         {\r
 403                 st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
 404         }\r
 405         st.Step();\r
 406 \r
 407         m_ids.clear();\r
 408 \r
 409         while(st.RowReturned())\r
 410         {\r
 411                 st.ResultInt(0,id);\r
 412                 m_ids[id]=false;\r
 413                 st.Step();\r
 414         }\r
 415 }\r
 416 \r
 417 void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)\r
 418 {\r
 419         std::vector<std::string> parts;\r
 420         std::string indexstr="";\r
 421         std::string identityidstr="";\r
 422         std::string datestr="";\r
 423         FCPv2::Message newmessage;\r
 424 \r
 425         // get the new edition #\r
 426         StringFunctions::Split(message["RedirectURI"],"/",parts);\r
 427         //edition # is 2nd to last part\r
 428         if(parts.size()>2)\r
 429         {\r
 430                 indexstr=parts[parts.size()-2];\r
 431         }\r
 432 \r
 433         // get identityid\r
 434         parts.clear();\r
 435         StringFunctions::Split(message["Identifier"],"|",parts);\r
 436         if(parts.size()>1)\r
 437         {\r
 438                 identityidstr=parts[1];\r
 439         }\r
 440         if(parts.size()>4)\r
 441         {\r
 442                 datestr=parts[4];\r
 443         }\r
 444 \r
 445         newmessage.SetName("ClientGet");\r
 446         newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);\r
 447         newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];\r
 448         newmessage["ReturnType"]="direct";\r
 449         newmessage["MaxSize"]="1000000";\r
 450 \r
 451         m_fcp->Send(newmessage);\r
 452 \r
 453 }\r
 454 \r
 455 void MessageListRequester::StartRequest(const long &id)\r
 456 {\r
 457         Poco::DateTime now;\r
 458         FCPv2::Message message;\r
 459         std::string publickey;\r
 460         int index=0;\r
 461         std::string indexstr;\r
 462         std::string identityidstr;\r
 463 \r
 464         SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
 465         st.Bind(0,id);\r
 466         st.Step();\r
 467 \r
 468         if(st.RowReturned())\r
 469         {\r
 470                 st.ResultText(0,publickey);\r
 471 \r
 472                 now=Poco::Timestamp();\r
 473 \r
 474                 SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");\r
 475                 st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));\r
 476                 st2.Bind(1,id);\r
 477                 st2.Step();\r
 478 \r
 479                 index=0;\r
 480                 if(st2.RowReturned())\r
 481                 {\r
 482                         if(st2.ResultNull(0)==false)\r
 483                         {\r
 484                                 st2.ResultInt(0,index);\r
 485                                 // don't increment index here - the node will let us know if there is a new edition\r
 486                                 // 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index\r
 487                                 index++;\r
 488                         }\r
 489                 }\r
 490                 st2.Finalize();\r
 491 \r
 492                 StringFunctions::Convert(index,indexstr);\r
 493                 StringFunctions::Convert(id,identityidstr);\r
 494 \r
 495                 message.SetName("ClientGet");\r
 496                 message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";\r
 497                 message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];\r
 498                 message["ReturnType"]="direct";\r
 499                 message["MaxSize"]="1000000";\r
 500 \r
 501                 m_fcp->Send(message);\r
 502 \r
 503                 m_requesting.push_back(id);\r
 504         }\r
 505         st.Finalize();\r
 506 \r
 507         m_ids[id]=true;\r
 508 }\r