src/freenet/messagelistrequester.cpp

   1 #include "../../include/freenet/messagelistrequester.h"\r
   2 #include "../../include/freenet/messagelistxml.h"\r
   3 \r
   4 #include <Poco/DateTime.h>\r
   5 #include <Poco/DateTimeFormatter.h>\r
   6 #include <Poco/Timestamp.h>\r
   7 \r
   8 #ifdef XMEM\r
   9         #include <xmem.h>\r
  10 #endif\r
  11 \r
  12 MessageListRequester::MessageListRequester()\r
  13 {\r
  14         Initialize();\r
  15 }\r
  16 \r
  17 MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester<long>(fcp)\r
  18 {\r
  19         Initialize();\r
  20 }\r
  21 \r
  22 void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)\r
  23 {\r
  24         SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
  25         st.Step();\r
  26         while(st.RowReturned())\r
  27         {\r
  28                 std::string boardname="";\r
  29                 std::string tempval="";\r
  30                 st.ResultText(0,boardname);\r
  31                 st.ResultText(1,tempval);\r
  32 \r
  33                 if(tempval=="true")\r
  34                 {\r
  35                         boards[boardname]=true;\r
  36                 }\r
  37                 else\r
  38                 {\r
  39                         boards[boardname]=false;\r
  40                 }\r
  41 \r
  42                 st.Step();\r
  43         }\r
  44 }\r
  45 \r
  46 const bool MessageListRequester::HandleAllData(FCPMessage &message)\r
  47 {       \r
  48         SQLite3DB::Statement st;\r
  49         SQLite3DB::Statement trustst;\r
  50         std::vector<std::string> idparts;\r
  51         long datalength;\r
  52         std::vector<char> data;\r
  53         MessageListXML xml;\r
  54         long identityid;\r
  55         long index;\r
  56         std::map<std::string,bool> boards;      // list of boards and if we will save messages for that board or not\r
  57         bool addmessage=false;\r
  58         std::string boardsstr="";\r
  59 \r
  60         GetBoardList(boards);\r
  61 \r
  62         StringFunctions::Split(message["Identifier"],"|",idparts);\r
  63         StringFunctions::Convert(message["DataLength"],datalength);\r
  64         StringFunctions::Convert(idparts[1],identityid);\r
  65         StringFunctions::Convert(idparts[2],index);\r
  66 \r
  67         // wait for all data to be received from connection\r
  68         while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()<datalength)\r
  69         {\r
  70                 m_fcp->Update(1);\r
  71         }\r
  72 \r
  73         // if we got disconnected- return immediately\r
  74         if(m_fcp->Connected()==false)\r
  75         {\r
  76                 return false;\r
  77         }\r
  78 \r
  79         // receive the file\r
  80         data.resize(datalength);\r
  81         m_fcp->ReceiveRaw(&data[0],datalength);\r
  82 \r
  83         // parse file into xml and update the database\r
  84         if(xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
  85         {\r
  86 \r
  87                 SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
  88                 SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
  89                 SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");\r
  90                 for(long i=0; i<xml.MessageCount(); i++)\r
  91                 {\r
  92 \r
  93                         // go through each board the message was posted to and see if we are saving messages to that board\r
  94                         // if the board isn't found, see if we are saving messages to new boards\r
  95                         boardsstr="";\r
  96                         addmessage=false;\r
  97                         std::vector<std::string> messageboards=xml.GetBoards(i);\r
  98                         for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
  99                         {\r
 100                                 if(boards.find((*j))!=boards.end())\r
 101                                 {\r
 102                                         if(boards[(*j)]==true)\r
 103                                         {\r
 104                                                 addmessage=true;\r
 105                                         }\r
 106                                 }\r
 107                                 else if(m_savetonewboards==true)\r
 108                                 {\r
 109                                         addmessage=true;\r
 110                                 }\r
 111                                 if(j!=messageboards.begin())\r
 112                                 {\r
 113                                         boardsstr+=", ";\r
 114                                 }\r
 115                                 boardsstr+=(*j);\r
 116                         }\r
 117 \r
 118                         if(addmessage==true)\r
 119                         {\r
 120                                 st.Bind(0,identityid);\r
 121                                 st.Bind(1,xml.GetDate(i));\r
 122                                 st.Bind(2,xml.GetIndex(i));\r
 123                                 st.Step();\r
 124                                 if(st.RowReturned()==false)\r
 125                                 {\r
 126                                         mst.Bind(0,identityid);\r
 127                                         mst.Bind(1,xml.GetDate(i));\r
 128                                         mst.Bind(2,xml.GetIndex(i));\r
 129                                         mst.Step();\r
 130                                         mst.Reset();\r
 131                                 }\r
 132                                 st.Reset();\r
 133                         }\r
 134                         else\r
 135                         {\r
 136                                 m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr);\r
 137                         }\r
 138                 }\r
 139 \r
 140                 // insert external message indexes\r
 141                 for(long i=0; i<xml.ExternalMessageCount(); i++)\r
 142                 {\r
 143                         if(xml.GetExternalType(i)=="Keyed")\r
 144                         {\r
 145                                 // go through each board the message was posted to and see if we are saving messages to that board\r
 146                                 // if the board isn't found, see if we are saving messages to new boards\r
 147                                 boardsstr="";\r
 148                                 addmessage=false;\r
 149                                 std::vector<std::string> messageboards=xml.GetExternalBoards(i);\r
 150                                 for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
 151                                 {\r
 152                                         if(boards.find((*j))!=boards.end())\r
 153                                         {\r
 154                                                 if(boards[(*j)]==true)\r
 155                                                 {\r
 156                                                         addmessage=true;\r
 157                                                 }\r
 158                                         }\r
 159                                         else if(m_savetonewboards==true)\r
 160                                         {\r
 161                                                 addmessage=true;\r
 162                                         }\r
 163                                         if(j!=messageboards.begin())\r
 164                                         {\r
 165                                                 boardsstr+=", ";\r
 166                                         }\r
 167                                         boardsstr+=(*j);\r
 168                                 }\r
 169 \r
 170                                 if(addmessage==true)\r
 171                                 {\r
 172                                         spk.Bind(0,xml.GetExternalIdentity(i));\r
 173                                         spk.Step();\r
 174                                         if(spk.RowReturned())\r
 175                                         {\r
 176                                                 int thisidentityid=0;\r
 177                                                 spk.ResultInt(0,thisidentityid);\r
 178                                                 mst.Bind(0,thisidentityid);\r
 179                                                 mst.Bind(1,xml.GetExternalDate(i));\r
 180                                                 mst.Bind(2,xml.GetExternalIndex(i));\r
 181                                                 mst.Step();\r
 182                                                 mst.Reset();\r
 183                                         }\r
 184                                         spk.Reset();\r
 185                                 }\r
 186                                 else\r
 187                                 {\r
 188                                         m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i));\r
 189                                 }\r
 190                         }\r
 191                 }\r
 192 \r
 193                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");\r
 194                 st.Bind(0,identityid);\r
 195                 st.Bind(1,idparts[4]);\r
 196                 st.Bind(2,index);\r
 197                 st.Step();\r
 198                 st.Finalize();\r
 199 \r
 200                 m_log->debug("MessageListRequester::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
 201         }\r
 202         else\r
 203         {\r
 204                 // bad data - mark index\r
 205                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
 206                 st.Bind(0,identityid);\r
 207                 st.Bind(1,idparts[4]);\r
 208                 st.Bind(2,index);\r
 209                 st.Step();\r
 210                 st.Finalize();\r
 211 \r
 212                 m_log->error("MessageListRequester::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);\r
 213         }\r
 214 \r
 215         // remove this identityid from request list\r
 216         RemoveFromRequestList(identityid);\r
 217 \r
 218         return true;\r
 219 \r
 220 }\r
 221 \r
 222 const bool MessageListRequester::HandleGetFailed(FCPMessage &message)\r
 223 {\r
 224         SQLite3DB::Statement st;\r
 225         std::vector<std::string> idparts;\r
 226         long identityid;\r
 227         long index;\r
 228 \r
 229         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 230         StringFunctions::Convert(idparts[1],identityid);\r
 231         StringFunctions::Convert(idparts[2],index);     \r
 232 \r
 233         // code 27 - permanent redirect\r
 234         if(message["Code"]=="27")\r
 235         {\r
 236                 StartRedirectRequest(message);\r
 237                 return true;\r
 238         }\r
 239 \r
 240         // if this is a fatal error - insert index into database so we won't try to download this index again\r
 241         if(message["Fatal"]=="true")\r
 242         {\r
 243                 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");\r
 244                 st.Bind(0,identityid);\r
 245                 st.Bind(1,idparts[4]);\r
 246                 st.Bind(2,index);\r
 247                 st.Step();\r
 248                 st.Finalize();\r
 249 \r
 250                 m_log->error("MessageListRequester::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);\r
 251         }\r
 252 \r
 253         // remove this identityid from request list\r
 254         RemoveFromRequestList(identityid);\r
 255 \r
 256         return true;\r
 257 }\r
 258 \r
 259 void MessageListRequester::Initialize()\r
 260 {\r
 261         m_fcpuniquename="MessageListRequester";\r
 262         std::string tempval;\r
 263         Option::Instance()->Get("MaxMessageListRequests",tempval);\r
 264         StringFunctions::Convert(tempval,m_maxrequests);\r
 265         if(m_maxrequests<1)\r
 266         {\r
 267                 m_maxrequests=1;\r
 268                 m_log->error("Option MaxMessageListRequests is currently set at "+tempval+".  It must be 1 or greater.");\r
 269         }\r
 270         if(m_maxrequests>100)\r
 271         {\r
 272                 m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+".  This value might be incorrectly configured.");\r
 273         }\r
 274 \r
 275         tempval="";\r
 276         Option::Instance()->Get("LocalTrustOverridesPeerTrust",tempval);\r
 277         if(tempval=="true")\r
 278         {\r
 279                 m_localtrustoverrides=true;\r
 280         }\r
 281         else\r
 282         {\r
 283                 m_localtrustoverrides=false;\r
 284         }\r
 285 \r
 286         tempval="";\r
 287         Option::Instance()->Get("SaveMessagesFromNewBoards",tempval);\r
 288         if(tempval=="true")\r
 289         {\r
 290                 m_savetonewboards=true;\r
 291         }\r
 292         else\r
 293         {\r
 294                 m_savetonewboards=false;\r
 295         }\r
 296 \r
 297 }\r
 298 \r
 299 void MessageListRequester::PopulateIDList()\r
 300 {\r
 301         Poco::DateTime date;\r
 302         int id;\r
 303 \r
 304         SQLite3DB::Statement st;\r
 305 \r
 306         // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
 307         if(m_localtrustoverrides==false)\r
 308         {\r
 309                 st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
 310         }\r
 311         else\r
 312         {\r
 313                 st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
 314         }\r
 315         st.Step();\r
 316 \r
 317         m_ids.clear();\r
 318 \r
 319         while(st.RowReturned())\r
 320         {\r
 321                 st.ResultInt(0,id);\r
 322                 m_ids[id]=false;\r
 323                 st.Step();\r
 324         }\r
 325 }\r
 326 \r
 327 void MessageListRequester::StartRedirectRequest(FCPMessage &message)\r
 328 {\r
 329         std::vector<std::string> parts;\r
 330         std::string indexstr="";\r
 331         std::string identityidstr="";\r
 332         std::string datestr="";\r
 333         FCPMessage newmessage;\r
 334 \r
 335         // get the new edition #\r
 336         StringFunctions::Split(message["RedirectURI"],"/",parts);\r
 337         //edition # is 2nd to last part\r
 338         if(parts.size()>2)\r
 339         {\r
 340                 indexstr=parts[parts.size()-2];\r
 341         }\r
 342 \r
 343         // get identityid\r
 344         parts.clear();\r
 345         StringFunctions::Split(message["Identifier"],"|",parts);\r
 346         if(parts.size()>1)\r
 347         {\r
 348                 identityidstr=parts[1];\r
 349         }\r
 350         if(parts.size()>4)\r
 351         {\r
 352                 datestr=parts[4];\r
 353         }\r
 354 \r
 355         newmessage.SetName("ClientGet");\r
 356         newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);\r
 357         newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];\r
 358         newmessage["ReturnType"]="direct";\r
 359         newmessage["MaxSize"]="1000000";\r
 360 \r
 361         m_fcp->SendMessage(newmessage);\r
 362 \r
 363 }\r
 364 \r
 365 void MessageListRequester::StartRequest(const long &id)\r
 366 {\r
 367         Poco::DateTime now;\r
 368         FCPMessage message;\r
 369         std::string publickey;\r
 370         int index=0;\r
 371         std::string indexstr;\r
 372         std::string identityidstr;\r
 373 \r
 374         SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
 375         st.Bind(0,id);\r
 376         st.Step();\r
 377 \r
 378         if(st.RowReturned())\r
 379         {\r
 380                 st.ResultText(0,publickey);\r
 381 \r
 382                 now=Poco::Timestamp();\r
 383 \r
 384                 SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");\r
 385                 st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));\r
 386                 st2.Bind(1,id);\r
 387                 st2.Step();\r
 388 \r
 389                 index=0;\r
 390                 if(st2.RowReturned())\r
 391                 {\r
 392                         if(st2.ResultNull(0)==false)\r
 393                         {\r
 394                                 st2.ResultInt(0,index);\r
 395                                 // don't increment index here - the node will let us know if there is a new edition\r
 396                                 // 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index\r
 397                                 index++;\r
 398                         }\r
 399                 }\r
 400                 st2.Finalize();\r
 401 \r
 402                 StringFunctions::Convert(index,indexstr);\r
 403                 StringFunctions::Convert(id,identityidstr);\r
 404 \r
 405                 message.SetName("ClientGet");\r
 406                 message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";\r
 407                 message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];\r
 408                 message["ReturnType"]="direct";\r
 409                 message["MaxSize"]="1000000";\r
 410 \r
 411                 m_fcp->SendMessage(message);\r
 412 \r
 413                 m_requesting.push_back(id);\r
 414         }\r
 415         st.Finalize();\r
 416 \r
 417         m_ids[id]=true;\r
 418 }\r