1 #include "../../include/freenet/messagelistrequester.h"
\r
2 #include "../../include/freenet/messagelistxml.h"
\r
4 #include <Poco/DateTimeFormatter.h>
\r
5 #include <Poco/DateTimeParser.h>
\r
6 #include <Poco/Timestamp.h>
\r
12 MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)
\r
17 MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)
\r
22 const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) const
\r
24 std::vector<std::string> dateparts;
\r
28 Poco::DateTime today;
\r
30 StringFunctions::Split(datestr,"-",dateparts);
\r
31 if(dateparts.size()==3)
\r
33 StringFunctions::Convert(dateparts[0],year);
\r
34 StringFunctions::Convert(dateparts[1],month);
\r
35 StringFunctions::Convert(dateparts[2],day);
\r
36 if(today.year()>year || (today.year()==year && today.month()>month) || (today.year()==year && today.month()==month && today.day()>=day))
\r
52 const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &datestr) const
\r
54 Poco::DateTime checkdate;
\r
55 Poco::DateTime date;
\r
57 if(Poco::DateTimeParser::tryParse(datestr,date,tzdiff))
\r
59 checkdate-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);
\r
75 void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards, const bool forceload)
\r
77 // only query database when forced, or an 30 minutes have passed since last query
\r
78 if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime())
\r
80 m_boardscache.clear();
\r
81 SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");
\r
83 while(st.RowReturned())
\r
85 std::string boardname="";
\r
86 std::string tempval="";
\r
87 st.ResultText(0,boardname);
\r
88 st.ResultText(1,tempval);
\r
92 m_boardscache[boardname]=true;
\r
96 m_boardscache[boardname]=false;
\r
101 m_boardscacheupdate=Poco::DateTime();
\r
104 boards=m_boardscache;
\r
108 const bool MessageListRequester::HandleAllData(FCPv2::Message &message)
\r
110 SQLite3DB::Statement st;
\r
111 std::vector<std::string> idparts;
\r
113 std::vector<char> data;
\r
114 MessageListXML xml;
\r
116 long fromidentityid;
\r
118 std::map<std::string,bool> boards; // list of boards and if we will save messages for that board or not
\r
119 std::map<std::string,long> identityids; // list of identity public keys and their id in the database
\r
120 bool addmessage=false;
\r
121 std::string boardsstr="";
\r
122 std::string datestr="";
\r
123 std::vector<std::string> dateparts;
\r
125 GetBoardList(boards);
\r
127 StringFunctions::Split(message["Identifier"],"|",idparts);
\r
128 StringFunctions::Convert(message["DataLength"],datalength);
\r
129 StringFunctions::Convert(idparts[1],identityid);
\r
130 StringFunctions::Convert(idparts[2],index);
\r
132 fromidentityid=identityid;
\r
134 // wait for all data to be received from connection
\r
135 m_fcp->WaitForBytes(1000,datalength);
\r
137 // if we got disconnected- return immediately
\r
138 if(m_fcp->IsConnected()==false)
\r
143 // receive the file
\r
144 m_fcp->Receive(data,datalength);
\r
146 // parse file into xml and update the database
\r
147 if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)
\r
150 m_db->Execute("BEGIN;");
\r
152 SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");
\r
153 SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);");
\r
154 SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;");
\r
156 for(long i=0; i<xml.MessageCount(); i++)
\r
159 // go through each board the message was posted to and see if we are saving messages to that board
\r
160 // if the board isn't found, see if we are saving messages to new boards
\r
163 std::vector<std::string> messageboards=xml.GetBoards(i);
\r
164 for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)
\r
166 if(boards.find((*j))!=boards.end())
\r
168 if(boards[(*j)]==true)
\r
173 else if(m_savetonewboards==true)
\r
177 if(j!=messageboards.begin())
\r
184 if(CheckDateNotFuture(xml.GetDate(i))==false)
\r
187 m_log->error(m_fcpuniquename+"::HandleAllData date for message is in future! "+xml.GetDate(i));
\r
190 if(addmessage==true && CheckDateWithinMaxDays(xml.GetDate(i))==false)
\r
195 if(addmessage==true)
\r
197 mst.Bind(0,identityid);
\r
198 mst.Bind(1,xml.GetDate(i));
\r
199 mst.Bind(2,xml.GetIndex(i));
\r
200 mst.Bind(3,identityid);
\r
204 // We need to update ID here, in case this index was already inserted from another
\r
205 // identity's message list. This doesn't reset try count - maybe we should if the from
\r
206 // identity was another identity
\r
207 ust.Bind(0,identityid);
\r
208 ust.Bind(1,identityid);
\r
209 ust.Bind(2,xml.GetDate(i));
\r
210 ust.Bind(3,xml.GetIndex(i));
\r
214 m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i));
\r
219 //m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr+" on "+xml.GetDate(i));
\r
223 // insert external message indexes
\r
224 for(long i=0; i<xml.ExternalMessageCount(); i++)
\r
226 if(xml.GetExternalType(i)=="Keyed")
\r
228 // go through each board the message was posted to and see if we are saving messages to that board
\r
229 // if the board isn't found, see if we are saving messages to new boards
\r
232 std::vector<std::string> messageboards=xml.GetExternalBoards(i);
\r
233 for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)
\r
235 if(boards.find((*j))!=boards.end())
\r
237 if(boards[(*j)]==true)
\r
242 else if(m_savetonewboards==true)
\r
246 if(j!=messageboards.begin())
\r
253 if(CheckDateNotFuture(xml.GetExternalDate(i))==false)
\r
256 m_log->error(m_fcpuniquename+"::HandleAllData date for external message is in future! "+xml.GetExternalDate(i));
\r
259 if(addmessage==true && CheckDateWithinMaxDays(xml.GetExternalDate(i))==false)
\r
264 if(addmessage==true)
\r
266 int thisidentityid=0;
\r
267 if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end())
\r
269 thisidentityid=identityids[xml.GetExternalIdentity(i)];
\r
273 spk.Bind(0,xml.GetExternalIdentity(i));
\r
276 if(spk.RowReturned())
\r
278 spk.ResultInt(0,thisidentityid);
\r
279 identityids[xml.GetExternalIdentity(i)]=thisidentityid;
\r
285 if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end())
\r
287 mst.Bind(0,thisidentityid);
\r
288 mst.Bind(1,xml.GetExternalDate(i));
\r
289 mst.Bind(2,xml.GetExternalIndex(i));
\r
290 mst.Bind(3,fromidentityid);
\r
294 m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i));
\r
299 //m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i) + " on " + xml.GetExternalDate(i));
\r
304 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");
\r
305 st.Bind(0,identityid);
\r
306 st.Bind(1,idparts[4]);
\r
315 m_db->Execute("COMMIT;");
\r
317 m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);
\r
321 // bad data - mark index
\r
322 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");
\r
323 st.Bind(0,identityid);
\r
324 st.Bind(1,idparts[4]);
\r
329 m_log->error(m_fcpuniquename+"::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);
\r
332 // remove this identityid from request list
\r
333 RemoveFromRequestList(identityid);
\r
335 // keep 2 days of request indexes in the cache
\r
336 while(m_requestindexcache.size()>2)
\r
338 m_requestindexcache.erase(m_requestindexcache.begin());
\r
345 const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)
\r
347 SQLite3DB::Statement st;
\r
348 std::vector<std::string> idparts;
\r
352 StringFunctions::Split(message["Identifier"],"|",idparts);
\r
353 StringFunctions::Convert(idparts[1],identityid);
\r
354 StringFunctions::Convert(idparts[2],index);
\r
356 // code 27 - permanent redirect
\r
357 if(message["Code"]=="27")
\r
359 StartRedirectRequest(message);
\r
363 // if this is a fatal error - insert index into database so we won't try to download this index again
\r
364 if(message["Fatal"]=="true")
\r
366 st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');");
\r
367 st.Bind(0,identityid);
\r
368 st.Bind(1,idparts[4]);
\r
373 m_log->error(m_fcpuniquename+"::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);
\r
376 // remove this identityid from request list
\r
377 RemoveFromRequestList(identityid);
\r
382 void MessageListRequester::Initialize()
\r
384 m_fcpuniquename="ActiveMessageListRequester";
\r
385 std::string tempval("");
\r
387 Option option(m_db);
\r
389 option.GetInt("MaxMessageListRequests",m_maxrequests);
\r
391 // active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2
\r
392 m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);
\r
394 if(m_maxrequests<1)
\r
397 m_log->error("Option MaxMessageListRequests is currently set at "+tempval+". It must be 1 or greater.");
\r
399 if(m_maxrequests>100)
\r
401 m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+". This value might be incorrectly configured.");
\r
405 option.Get("LocalTrustOverridesPeerTrust",tempval);
\r
406 if(tempval=="true")
\r
408 m_localtrustoverrides=true;
\r
412 m_localtrustoverrides=false;
\r
416 option.Get("SaveMessagesFromNewBoards",tempval);
\r
417 if(tempval=="true")
\r
419 m_savetonewboards=true;
\r
423 m_savetonewboards=false;
\r
426 m_messagedownloadmaxdaysbackward=5;
\r
428 option.Get("MessageDownloadMaxDaysBackward",tempval);
\r
429 StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);
\r
431 m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0);
\r
435 void MessageListRequester::PopulateIDList()
\r
437 Poco::DateTime date;
\r
438 Poco::DateTime yesterday=date-Poco::Timespan(1,0,0,0,0);
\r
441 SQLite3DB::Statement st;
\r
443 // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)
\r
444 if(m_localtrustoverrides==false)
\r
446 st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
\r
450 st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
\r
456 while(st.RowReturned())
\r
458 st.ResultInt(0,id);
\r
464 void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)
\r
466 std::vector<std::string> parts;
\r
467 std::string indexstr="";
\r
468 std::string identityidstr="";
\r
469 std::string datestr="";
\r
470 FCPv2::Message newmessage;
\r
472 // get the new edition #
\r
473 StringFunctions::Split(message["RedirectURI"],"/",parts);
\r
474 //edition # is 2nd to last part
\r
477 indexstr=parts[parts.size()-2];
\r
482 StringFunctions::Split(message["Identifier"],"|",parts);
\r
485 identityidstr=parts[1];
\r
492 newmessage.SetName("ClientGet");
\r
493 newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);
\r
494 newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];
\r
495 newmessage["ReturnType"]="direct";
\r
496 newmessage["MaxSize"]="1000000";
\r
498 m_fcp->Send(newmessage);
\r
502 void MessageListRequester::StartRequest(const long &id)
\r
504 Poco::DateTime now;
\r
505 FCPv2::Message message;
\r
506 std::string publickey;
\r
508 std::string indexstr;
\r
509 std::string identityidstr;
\r
511 SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");
\r
515 if(st.RowReturned())
\r
517 st.ResultText(0,publickey);
\r
519 now=Poco::Timestamp();
\r
521 SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");
\r
522 st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));
\r
527 if(st2.RowReturned())
\r
529 if(st2.ResultNull(0)==false)
\r
531 st2.ResultInt(0,index);
\r
532 // don't increment index here - the node will let us know if there is a new edition
\r
533 // 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index
\r
539 StringFunctions::Convert(index,indexstr);
\r
540 StringFunctions::Convert(id,identityidstr);
\r
542 message.SetName("ClientGet");
\r
543 message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";
\r
544 message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];
\r
545 message["ReturnType"]="direct";
\r
546 message["MaxSize"]="1000000";
\r
548 m_fcp->Send(message);
\r
550 m_requesting.push_back(id);
\r