X-Git-Url: https://git.pterodactylus.net/?a=blobdiff_plain;f=src%2Ffreenet%2Fmessagelistrequester.cpp;h=e54e0f232c46f891b1c9c6e1828d07a2c2a8a163;hb=ed0732b2550c23c05fc9faf925620e87ee6dee12;hp=24f70d1c818cc1debd4aabd015c762f6e3e1078f;hpb=b88f50bfec6dbcd169bb8285e7c42b93baf52b6b;p=fms.git diff --git a/src/freenet/messagelistrequester.cpp b/src/freenet/messagelistrequester.cpp index 24f70d1..e54e0f2 100644 --- a/src/freenet/messagelistrequester.cpp +++ b/src/freenet/messagelistrequester.cpp @@ -1,20 +1,20 @@ #include "../../include/freenet/messagelistrequester.h" #include "../../include/freenet/messagelistxml.h" -#include #include +#include #include #ifdef XMEM #include #endif -MessageListRequester::MessageListRequester() +MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester(db) { Initialize(); } -MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester(fcp) +MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester(db,fcp) { Initialize(); } @@ -49,41 +49,74 @@ const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) } -void MessageListRequester::GetBoardList(std::map &boards) +const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &datestr) const { - SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;"); - st.Step(); - while(st.RowReturned()) + Poco::DateTime checkdate; + Poco::DateTime date; + int tzdiff=0; + if(Poco::DateTimeParser::tryParse(datestr,date,tzdiff)) { - std::string boardname=""; - std::string tempval=""; - st.ResultText(0,boardname); - st.ResultText(1,tempval); - - if(tempval=="true") + checkdate-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0); + if(checkdate<=date) { - boards[boardname]=true; + return true; } else { - boards[boardname]=false; + return false; } + } + else + { + return false; + } +} +void MessageListRequester::GetBoardList(std::map &boards, const bool forceload) +{ + // only query database when forced, or an 30 minutes have passed since last query + if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime()) + { + m_boardscache.clear(); + SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;"); st.Step(); + while(st.RowReturned()) + { + std::string boardname=""; + std::string tempval=""; + st.ResultText(0,boardname); + st.ResultText(1,tempval); + + if(tempval=="true") + { + m_boardscache[boardname]=true; + } + else + { + m_boardscache[boardname]=false; + } + + st.Step(); + } + m_boardscacheupdate=Poco::DateTime(); } + + boards=m_boardscache; + } -const bool MessageListRequester::HandleAllData(FCPMessage &message) +const bool MessageListRequester::HandleAllData(FCPv2::Message &message) { SQLite3DB::Statement st; - SQLite3DB::Statement trustst; std::vector idparts; long datalength; std::vector data; MessageListXML xml; long identityid; + long fromidentityid; long index; std::map boards; // list of boards and if we will save messages for that board or not + std::map identityids; // list of identity public keys and their id in the database bool addmessage=false; std::string boardsstr=""; std::string datestr=""; @@ -96,29 +129,30 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message) StringFunctions::Convert(idparts[1],identityid); StringFunctions::Convert(idparts[2],index); + fromidentityid=identityid; + // wait for all data to be received from connection - while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()Update(1); - } + m_fcp->WaitForBytes(1000,datalength); // if we got disconnected- return immediately - if(m_fcp->Connected()==false) + if(m_fcp->IsConnected()==false) { return false; } // receive the file - data.resize(datalength); - m_fcp->ReceiveRaw(&data[0],datalength); + m_fcp->Receive(data,datalength); // parse file into xml and update the database - if(xml.ParseXML(std::string(data.begin(),data.end()))==true) + if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true) { - SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;"); + m_db->Execute("BEGIN;"); + SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;"); - SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');"); + SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);"); + SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;"); + for(long i=0; ierror("MessageListRequester::HandleAllData date for message is in future! "+xml.GetDate(i)); + m_log->error(m_fcpuniquename+"::HandleAllData date for message is in future! "+xml.GetDate(i)); + } + + if(addmessage==true && CheckDateWithinMaxDays(xml.GetDate(i))==false) + { + addmessage=false; } if(addmessage==true) { - st.Bind(0,identityid); - st.Bind(1,xml.GetDate(i)); - st.Bind(2,xml.GetIndex(i)); - st.Step(); - if(st.RowReturned()==false) - { - mst.Bind(0,identityid); - mst.Bind(1,xml.GetDate(i)); - mst.Bind(2,xml.GetIndex(i)); - mst.Step(); - mst.Reset(); - } - st.Reset(); + mst.Bind(0,identityid); + mst.Bind(1,xml.GetDate(i)); + mst.Bind(2,xml.GetIndex(i)); + mst.Bind(3,identityid); + mst.Step(); + mst.Reset(); + + // We need to update ID here, in case this index was already inserted from another + // identity's message list. This doesn't reset try count - maybe we should if the from + // identity was another identity + ust.Bind(0,identityid); + ust.Bind(1,identityid); + ust.Bind(2,xml.GetDate(i)); + ust.Bind(3,xml.GetIndex(i)); + ust.Step(); + ust.Reset(); + + m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i)); + } else { - m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr); + //m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr+" on "+xml.GetDate(i)); } } @@ -208,28 +253,50 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message) if(CheckDateNotFuture(xml.GetExternalDate(i))==false) { addmessage=false; - m_log->error("MessageListRequester::HandleAllData date for external message is in future! "+xml.GetExternalDate(i)); + m_log->error(m_fcpuniquename+"::HandleAllData date for external message is in future! "+xml.GetExternalDate(i)); + } + + if(addmessage==true && CheckDateWithinMaxDays(xml.GetExternalDate(i))==false) + { + addmessage=false; } if(addmessage==true) { - spk.Bind(0,xml.GetExternalIdentity(i)); - spk.Step(); - if(spk.RowReturned()) + int thisidentityid=0; + if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end()) + { + thisidentityid=identityids[xml.GetExternalIdentity(i)]; + } + else + { + spk.Bind(0,xml.GetExternalIdentity(i)); + spk.Step(); + + if(spk.RowReturned()) + { + spk.ResultInt(0,thisidentityid); + identityids[xml.GetExternalIdentity(i)]=thisidentityid; + } + + spk.Reset(); + } + + if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end()) { - int thisidentityid=0; - spk.ResultInt(0,thisidentityid); mst.Bind(0,thisidentityid); mst.Bind(1,xml.GetExternalDate(i)); mst.Bind(2,xml.GetExternalIndex(i)); + mst.Bind(3,fromidentityid); mst.Step(); mst.Reset(); + + m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i)); } - spk.Reset(); } else { - m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i)); + //m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i) + " on " + xml.GetExternalDate(i)); } } } @@ -241,7 +308,13 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message) st.Step(); st.Finalize(); - m_log->debug("MessageListRequester::HandleAllData parsed MessageList XML file : "+message["Identifier"]); + spk.Finalize(); + mst.Finalize(); + ust.Finalize(); + + m_db->Execute("COMMIT;"); + + m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]); } else { @@ -253,17 +326,23 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message) st.Step(); st.Finalize(); - m_log->error("MessageListRequester::HandleAllData error parsing MessageList XML file : "+message["Identifier"]); + m_log->error(m_fcpuniquename+"::HandleAllData error parsing MessageList XML file : "+message["Identifier"]); } // remove this identityid from request list RemoveFromRequestList(identityid); + // keep 2 days of request indexes in the cache + while(m_requestindexcache.size()>2) + { + m_requestindexcache.erase(m_requestindexcache.begin()); + } + return true; } -const bool MessageListRequester::HandleGetFailed(FCPMessage &message) +const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message) { SQLite3DB::Statement st; std::vector idparts; @@ -291,7 +370,7 @@ const bool MessageListRequester::HandleGetFailed(FCPMessage &message) st.Step(); st.Finalize(); - m_log->error("MessageListRequester::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]); + m_log->error(m_fcpuniquename+"::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]); } // remove this identityid from request list @@ -302,10 +381,16 @@ const bool MessageListRequester::HandleGetFailed(FCPMessage &message) void MessageListRequester::Initialize() { - m_fcpuniquename="MessageListRequester"; - std::string tempval; - Option::Instance()->Get("MaxMessageListRequests",tempval); - StringFunctions::Convert(tempval,m_maxrequests); + m_fcpuniquename="ActiveMessageListRequester"; + std::string tempval(""); + m_maxrequests=0; + Option option(m_db); + + option.GetInt("MaxMessageListRequests",m_maxrequests); + + // active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2 + m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2); + if(m_maxrequests<1) { m_maxrequests=1; @@ -317,7 +402,7 @@ void MessageListRequester::Initialize() } tempval=""; - Option::Instance()->Get("LocalTrustOverridesPeerTrust",tempval); + option.Get("LocalTrustOverridesPeerTrust",tempval); if(tempval=="true") { m_localtrustoverrides=true; @@ -328,7 +413,7 @@ void MessageListRequester::Initialize() } tempval=""; - Option::Instance()->Get("SaveMessagesFromNewBoards",tempval); + option.Get("SaveMessagesFromNewBoards",tempval); if(tempval=="true") { m_savetonewboards=true; @@ -338,11 +423,19 @@ void MessageListRequester::Initialize() m_savetonewboards=false; } + m_messagedownloadmaxdaysbackward=5; + tempval="5"; + option.Get("MessageDownloadMaxDaysBackward",tempval); + StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward); + + m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0); + } void MessageListRequester::PopulateIDList() { Poco::DateTime date; + Poco::DateTime yesterday=date-Poco::Timespan(1,0,0,0,0); int id; SQLite3DB::Statement st; @@ -350,11 +443,11 @@ void MessageListRequester::PopulateIDList() // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) if(m_localtrustoverrides==false) { - st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); } else { - st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); } st.Step(); @@ -368,13 +461,13 @@ void MessageListRequester::PopulateIDList() } } -void MessageListRequester::StartRedirectRequest(FCPMessage &message) +void MessageListRequester::StartRedirectRequest(FCPv2::Message &message) { std::vector parts; std::string indexstr=""; std::string identityidstr=""; std::string datestr=""; - FCPMessage newmessage; + FCPv2::Message newmessage; // get the new edition # StringFunctions::Split(message["RedirectURI"],"/",parts); @@ -402,14 +495,14 @@ void MessageListRequester::StartRedirectRequest(FCPMessage &message) newmessage["ReturnType"]="direct"; newmessage["MaxSize"]="1000000"; - m_fcp->SendMessage(newmessage); + m_fcp->Send(newmessage); } void MessageListRequester::StartRequest(const long &id) { Poco::DateTime now; - FCPMessage message; + FCPv2::Message message; std::string publickey; int index=0; std::string indexstr; @@ -452,7 +545,7 @@ void MessageListRequester::StartRequest(const long &id) message["ReturnType"]="direct"; message["MaxSize"]="1000000"; - m_fcp->SendMessage(message); + m_fcp->Send(message); m_requesting.push_back(id); }