src/freenet/messagelistrequester.cpp

   1 #include "../../include/freenet/messagelistrequester.h"\r
   2 #include "../../include/freenet/messagelistxml.h"\r
   3 \r
   4 #include <Poco/DateTime.h>\r
   5 #include <Poco/DateTimeFormatter.h>\r
   6 #include <Poco/Timestamp.h>\r
   7 \r
   8 #ifdef XMEM\r
   9         #include <xmem.h>\r
  10 #endif\r
  11 \r
  12 MessageListRequester::MessageListRequester()\r
  13 {\r
  14         Initialize();\r
  15 }\r
  16 \r
  17 MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester<long>(fcp)\r
  18 {\r
  19         Initialize();\r
  20 }\r
  21 \r
  22 const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) const\r
  23 {\r
  24         std::vector<std::string> dateparts;\r
  25         int year=0;\r
  26         int month=0;\r
  27         int day=0;\r
  28         Poco::DateTime today;\r
  29 \r
  30         StringFunctions::Split(datestr,"-",dateparts);\r
  31         if(dateparts.size()==3)\r
  32         {\r
  33                 StringFunctions::Convert(dateparts[0],year);\r
  34                 StringFunctions::Convert(dateparts[1],month);\r
  35                 StringFunctions::Convert(dateparts[2],day);\r
  36                 if(today.year()>year || (today.year()==year && today.month()>month) || (today.year()==year && today.month()==month && today.day()>=day))\r
  37                 {\r
  38                         return true;\r
  39                 }\r
  40                 else\r
  41                 {\r
  42                         return false;\r
  43                 }\r
  44         }\r
  45         else\r
  46         {\r
  47                 return false;\r
  48         }\r
  49 \r
  50 }\r
  51 \r
  52 void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)\r
  53 {\r
  54         SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
  55         st.Step();\r
  56         while(st.RowReturned())\r
  57         {\r
  58                 std::string boardname="";\r
  59                 std::string tempval="";\r
  60                 st.ResultText(0,boardname);\r
  61                 st.ResultText(1,tempval);\r
  62 \r
  63                 if(tempval=="true")\r
  64                 {\r
  65                         boards[boardname]=true;\r
  66                 }\r
  67                 else\r
  68                 {\r
  69                         boards[boardname]=false;\r
  70                 }\r
  71 \r
  72                 st.Step();\r
  73         }\r
  74 }\r
  75 \r
  76 const bool MessageListRequester::HandleAllData(FCPMessage &message)\r
  77 {       \r
  78         SQLite3DB::Statement st;\r
  79         SQLite3DB::Statement trustst;\r
  80         std::vector<std::string> idparts;\r
  81         long datalength;\r
  82         std::vector<char> data;\r
  83         MessageListXML xml;\r
  84         long identityid;\r
  85         long index;\r
  86         std::map<std::string,bool> boards;      // list of boards and if we will save messages for that board or not\r
  87         bool addmessage=false;\r
  88         std::string boardsstr="";\r
  89         std::string datestr="";\r
  90         std::vector<std::string> dateparts;\r
  91 \r
  92         GetBoardList(boards);\r
  93 \r
  94         StringFunctions::Split(message["Identifier"],"|",idparts);\r
  95         StringFunctions::Convert(message["DataLength"],datalength);\r
  96         StringFunctions::Convert(idparts[1],identityid);\r
  97         StringFunctions::Convert(idparts[2],index);\r
  98 \r
  99         // wait for all data to be received from connection\r
 100         while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()<datalength)\r
 101         {\r
 102                 m_fcp->Update(1);\r
 103         }\r
 104 \r
 105         // if we got disconnected- return immediately\r
 106         if(m_fcp->Connected()==false)\r
 107         {\r
 108                 return false;\r
 109         }\r
 110 \r
 111         // receive the file\r
 112         data.resize(datalength);\r
 113         if(data.size()>0)\r
 114         {\r
 115                 m_fcp->ReceiveRaw(&data[0],datalength);\r
 116         }\r
 117 \r
 118         // parse file into xml and update the database\r
 119         if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
 120         {\r
 121 \r
 122                 SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
 123                 SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
 124                 SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");\r
 125                 for(long i=0; i<xml.MessageCount(); i++)\r
 126                 {\r
 127 \r
 128                         // go through each board the message was posted to and see if we are saving messages to that board\r
 129                         // if the board isn't found, see if we are saving messages to new boards\r
 130                         boardsstr="";\r
 131                         addmessage=false;\r
 132                         std::vector<std::string> messageboards=xml.GetBoards(i);\r
 133                         for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
 134                         {\r
 135                                 if(boards.find((*j))!=boards.end())\r
 136                                 {\r
 137                                         if(boards[(*j)]==true)\r
 138                                         {\r
 139                                                 addmessage=true;\r
 140                                         }\r
 141                                 }\r
 142                                 else if(m_savetonewboards==true)\r
 143                                 {\r
 144                                         addmessage=true;\r
 145                                 }\r
 146                                 if(j!=messageboards.begin())\r
 147                                 {\r
 148                                         boardsstr+=", ";\r
 149                                 }\r
 150                                 boardsstr+=(*j);\r
 151                         }\r
 152 \r
 153                         if(CheckDateNotFuture(xml.GetDate(i))==false)\r
 154                         {\r
 155                                 addmessage=false;\r
 156                                 m_log->error("MessageListRequester::HandleAllData date for message is in future! "+xml.GetDate(i));\r
 157                         }\r
 158 \r
 159                         if(addmessage==true)\r
 160                         {\r
 161                                 st.Bind(0,identityid);\r
 162                                 st.Bind(1,xml.GetDate(i));\r
 163                                 st.Bind(2,xml.GetIndex(i));\r
 164                                 st.Step();\r
 165                                 if(st.RowReturned()==false)\r
 166                                 {\r
 167                                         mst.Bind(0,identityid);\r
 168                                         mst.Bind(1,xml.GetDate(i));\r
 169                                         mst.Bind(2,xml.GetIndex(i));\r
 170                                         mst.Step();\r
 171                                         mst.Reset();\r
 172                                 }\r
 173                                 st.Reset();\r
 174                         }\r
 175                         else\r
 176                         {\r
 177                                 m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr);\r
 178                         }\r
 179                 }\r
 180 \r
 181                 // insert external message indexes\r
 182                 for(long i=0; i<xml.ExternalMessageCount(); i++)\r
 183                 {\r
 184                         if(xml.GetExternalType(i)=="Keyed")\r
 185                         {\r
 186                                 // go through each board the message was posted to and see if we are saving messages to that board\r
 187                                 // if the board isn't found, see if we are saving messages to new boards\r
 188                                 boardsstr="";\r
 189                                 addmessage=false;\r
 190                                 std::vector<std::string> messageboards=xml.GetExternalBoards(i);\r
 191                                 for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
 192                                 {\r
 193                                         if(boards.find((*j))!=boards.end())\r
 194                                         {\r
 195                                                 if(boards[(*j)]==true)\r
 196                                                 {\r
 197                                                         addmessage=true;\r
 198                                                 }\r
 199                                         }\r
 200                                         else if(m_savetonewboards==true)\r
 201                                         {\r
 202                                                 addmessage=true;\r
 203                                         }\r
 204                                         if(j!=messageboards.begin())\r
 205                                         {\r
 206                                                 boardsstr+=", ";\r
 207                                         }\r
 208                                         boardsstr+=(*j);\r
 209                                 }\r
 210 \r
 211                                 if(CheckDateNotFuture(xml.GetExternalDate(i))==false)\r
 212                                 {\r
 213                                         addmessage=false;\r
 214                                         m_log->error("MessageListRequester::HandleAllData date for external message is in future! "+xml.GetExternalDate(i));\r
 215                                 }\r
 216 \r
 217                                 if(addmessage==true)\r
 218                                 {\r
 219                                         spk.Bind(0,xml.GetExternalIdentity(i));\r
 220                                         spk.Step();\r
 221                                         if(spk.RowReturned())\r
 222                                         {\r
 223                                                 int thisidentityid=0;\r
 224                                                 spk.ResultInt(0,thisidentityid);\r
 225                                                 mst.Bind(0,thisidentityid);\r
 226                                                 mst.Bind(1,xml.GetExternalDate(i));\r
 227                                                 mst.Bind(2,xml.GetExternalIndex(i));\r
 228                                                 mst.Step();\r
 229                                                 mst.Reset();\r
 230                                         }\r
 231                                         spk.Reset();\r
 232                                 }\r
 233                                 else\r
 234                                 {\r
 235                                         m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i));\r
 236                                 }\r
 237                         }\r
 238                 }\r
 239 \r
 240                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");\r
 241                 st.Bind(0,identityid);\r
 242                 st.Bind(1,idparts[4]);\r
 243                 st.Bind(2,index);\r
 244                 st.Step();\r
 245                 st.Finalize();\r
 246 \r
 247                 m_log->debug("MessageListRequester::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
 248         }\r
 249         else\r
 250         {\r
 251                 // bad data - mark index\r
 252                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
 253                 st.Bind(0,identityid);\r
 254                 st.Bind(1,idparts[4]);\r
 255                 st.Bind(2,index);\r
 256                 st.Step();\r
 257                 st.Finalize();\r
 258 \r
 259                 m_log->error("MessageListRequester::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);\r
 260         }\r
 261 \r
 262         // remove this identityid from request list\r
 263         RemoveFromRequestList(identityid);\r
 264 \r
 265         return true;\r
 266 \r
 267 }\r
 268 \r
 269 const bool MessageListRequester::HandleGetFailed(FCPMessage &message)\r
 270 {\r
 271         SQLite3DB::Statement st;\r
 272         std::vector<std::string> idparts;\r
 273         long identityid;\r
 274         long index;\r
 275 \r
 276         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 277         StringFunctions::Convert(idparts[1],identityid);\r
 278         StringFunctions::Convert(idparts[2],index);     \r
 279 \r
 280         // code 27 - permanent redirect\r
 281         if(message["Code"]=="27")\r
 282         {\r
 283                 StartRedirectRequest(message);\r
 284                 return true;\r
 285         }\r
 286 \r
 287         // if this is a fatal error - insert index into database so we won't try to download this index again\r
 288         if(message["Fatal"]=="true")\r
 289         {\r
 290                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
 291                 st.Bind(0,identityid);\r
 292                 st.Bind(1,idparts[4]);\r
 293                 st.Bind(2,index);\r
 294                 st.Step();\r
 295                 st.Finalize();\r
 296 \r
 297                 m_log->error("MessageListRequester::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);\r
 298         }\r
 299 \r
 300         // remove this identityid from request list\r
 301         RemoveFromRequestList(identityid);\r
 302 \r
 303         return true;\r
 304 }\r
 305 \r
 306 void MessageListRequester::Initialize()\r
 307 {\r
 308         m_fcpuniquename="MessageListRequester";\r
 309         std::string tempval="";\r
 310 \r
 311         m_maxrequests=0;\r
 312         Option::Instance()->GetInt("MaxMessageListRequests",m_maxrequests);\r
 313         if(m_maxrequests<1)\r
 314         {\r
 315                 m_maxrequests=1;\r
 316                 m_log->error("Option MaxMessageListRequests is currently set at "+tempval+".  It must be 1 or greater.");\r
 317         }\r
 318         if(m_maxrequests>100)\r
 319         {\r
 320                 m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+".  This value might be incorrectly configured.");\r
 321         }\r
 322 \r
 323         tempval="";\r
 324         Option::Instance()->Get("LocalTrustOverridesPeerTrust",tempval);\r
 325         if(tempval=="true")\r
 326         {\r
 327                 m_localtrustoverrides=true;\r
 328         }\r
 329         else\r
 330         {\r
 331                 m_localtrustoverrides=false;\r
 332         }\r
 333 \r
 334         tempval="";\r
 335         Option::Instance()->Get("SaveMessagesFromNewBoards",tempval);\r
 336         if(tempval=="true")\r
 337         {\r
 338                 m_savetonewboards=true;\r
 339         }\r
 340         else\r
 341         {\r
 342                 m_savetonewboards=false;\r
 343         }\r
 344 \r
 345 }\r
 346 \r
 347 void MessageListRequester::PopulateIDList()\r
 348 {\r
 349         Poco::DateTime date;\r
 350         int id;\r
 351 \r
 352         SQLite3DB::Statement st;\r
 353 \r
 354         // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
 355         if(m_localtrustoverrides==false)\r
 356         {\r
 357                 st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
 358         }\r
 359         else\r
 360         {\r
 361                 st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
 362         }\r
 363         st.Step();\r
 364 \r
 365         m_ids.clear();\r
 366 \r
 367         while(st.RowReturned())\r
 368         {\r
 369                 st.ResultInt(0,id);\r
 370                 m_ids[id]=false;\r
 371                 st.Step();\r
 372         }\r
 373 }\r
 374 \r
 375 void MessageListRequester::StartRedirectRequest(FCPMessage &message)\r
 376 {\r
 377         std::vector<std::string> parts;\r
 378         std::string indexstr="";\r
 379         std::string identityidstr="";\r
 380         std::string datestr="";\r
 381         FCPMessage newmessage;\r
 382 \r
 383         // get the new edition #\r
 384         StringFunctions::Split(message["RedirectURI"],"/",parts);\r
 385         //edition # is 2nd to last part\r
 386         if(parts.size()>2)\r
 387         {\r
 388                 indexstr=parts[parts.size()-2];\r
 389         }\r
 390 \r
 391         // get identityid\r
 392         parts.clear();\r
 393         StringFunctions::Split(message["Identifier"],"|",parts);\r
 394         if(parts.size()>1)\r
 395         {\r
 396                 identityidstr=parts[1];\r
 397         }\r
 398         if(parts.size()>4)\r
 399         {\r
 400                 datestr=parts[4];\r
 401         }\r
 402 \r
 403         newmessage.SetName("ClientGet");\r
 404         newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);\r
 405         newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];\r
 406         newmessage["ReturnType"]="direct";\r
 407         newmessage["MaxSize"]="1000000";\r
 408 \r
 409         m_fcp->SendMessage(newmessage);\r
 410 \r
 411 }\r
 412 \r
 413 void MessageListRequester::StartRequest(const long &id)\r
 414 {\r
 415         Poco::DateTime now;\r
 416         FCPMessage message;\r
 417         std::string publickey;\r
 418         int index=0;\r
 419         std::string indexstr;\r
 420         std::string identityidstr;\r
 421 \r
 422         SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
 423         st.Bind(0,id);\r
 424         st.Step();\r
 425 \r
 426         if(st.RowReturned())\r
 427         {\r
 428                 st.ResultText(0,publickey);\r
 429 \r
 430                 now=Poco::Timestamp();\r
 431 \r
 432                 SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");\r
 433                 st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));\r
 434                 st2.Bind(1,id);\r
 435                 st2.Step();\r
 436 \r
 437                 index=0;\r
 438                 if(st2.RowReturned())\r
 439                 {\r
 440                         if(st2.ResultNull(0)==false)\r
 441                         {\r
 442                                 st2.ResultInt(0,index);\r
 443                                 // don't increment index here - the node will let us know if there is a new edition\r
 444                                 // 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index\r
 445                                 index++;\r
 446                         }\r
 447                 }\r
 448                 st2.Finalize();\r
 449 \r
 450                 StringFunctions::Convert(index,indexstr);\r
 451                 StringFunctions::Convert(id,identityidstr);\r
 452 \r
 453                 message.SetName("ClientGet");\r
 454                 message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";\r
 455                 message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];\r
 456                 message["ReturnType"]="direct";\r
 457                 message["MaxSize"]="1000000";\r
 458 \r
 459                 m_fcp->SendMessage(message);\r
 460 \r
 461                 m_requesting.push_back(id);\r
 462         }\r
 463         st.Finalize();\r
 464 \r
 465         m_ids[id]=true;\r
 466 }\r