X-Git-Url: https://git.pterodactylus.net/?a=blobdiff_plain;f=src%2Ffreenet%2Ftrustlistrequester.cpp;h=96d24df96934a6dc96dbfd1be0e8def74813081c;hb=59a5414ec47a2932a7802fcd1d98c4d80166564f;hp=5b13a6d1ce8a3537fe50cab2ed90da111678a1e5;hpb=9048d8e482c91960265f29c2b5b3112f2a52f3d8;p=fms.git diff --git a/src/freenet/trustlistrequester.cpp b/src/freenet/trustlistrequester.cpp index 5b13a6d..96d24df 100644 --- a/src/freenet/trustlistrequester.cpp +++ b/src/freenet/trustlistrequester.cpp @@ -3,34 +3,26 @@ #include "../../include/stringfunctions.h" #include "../../include/freenet/trustlistxml.h" +#include +#include + #ifdef XMEM #include #endif -TrustListRequester::TrustListRequester() +TrustListRequester::TrustListRequester(SQLite3DB::DB *db):IIndexRequester(db) { Initialize(); } -TrustListRequester::TrustListRequester(FCPv2 *fcp):IFCPConnected(fcp) +TrustListRequester::TrustListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester(db,fcp) { Initialize(); } -void TrustListRequester::FCPConnected() -{ - m_requesting.clear(); - PopulateIDList(); -} - -void TrustListRequester::FCPDisconnected() -{ - -} - -const bool TrustListRequester::HandleAllData(FCPMessage &message) +const bool TrustListRequester::HandleAllData(FCPv2::Message &message) { - DateTime now; + Poco::DateTime now; SQLite3DB::Statement st; SQLite3DB::Statement trustst; std::vector idparts; @@ -39,32 +31,85 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) TrustListXML xml; long identityid; long index; + int insertcount=0; + int dayinsertcount=0; + int previnsertcount=0; - now.SetToGMTime(); StringFunctions::Split(message["Identifier"],"|",idparts); StringFunctions::Convert(message["DataLength"],datalength); StringFunctions::Convert(idparts[1],identityid); StringFunctions::Convert(idparts[2],index); // wait for all data to be received from connection - while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()Update(1); - } + m_fcp->WaitForBytes(1000,datalength); // if we got disconnected- return immediately - if(m_fcp->Connected()==false) + if(m_fcp->IsConnected()==false) { return false; } // receive the file - data.resize(datalength); - m_fcp->ReceiveRaw(&data[0],datalength); + m_fcp->Receive(data,datalength); + + // get count of identities added in last 24 hours + st=m_db->Prepare("SELECT COUNT(*) FROM tblIdentity WHERE DateAdded>=?;"); + now-=Poco::Timespan(1,0,0,0,0); + st.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")); + st.Step(); + if(st.RowReturned()) + { + if(st.ResultNull(0)==false) + { + st.ResultInt(0,dayinsertcount); + } + } + else + { + m_log->error("TrustListRequester::HandleAllData couldn't get count of identities added in last 24 hours"); + } + + // get count of identities added more than 24 hours ago - if 0 then we will accept more than 100 identities now + st=m_db->Prepare("SELECT COUNT(*) FROM tblIdentity WHERE DateAddederror("TrustListRequester::HandleAllData couldn't get count of identities added more than 24 hours ago"); + } + + now=Poco::DateTime(); // parse file into xml and update the database - if(xml.ParseXML(std::string(data.begin(),data.end()))==true) + if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true) { + // find the identity name and public key of the identity publishing the trust list + std::string publisherid=""; + st=m_db->Prepare("SELECT Name,PublicKey FROM tblIdentity WHERE IdentityID=?;"); + st.Bind(0,identityid); + st.Step(); + if(st.RowReturned()) + { + std::string publishername=""; + std::string publisherpublickey=""; + st.ResultText(0,publishername); + st.ResultText(1,publisherpublickey); + publisherid=publishername; + if(publisherpublickey.size()>4) + { + publisherid+=publisherpublickey.substr(3,44); + } + } + st.Finalize(); + + m_db->Execute("BEGIN;"); // drop all existing peer trust from this identity - we will rebuild it when we go through each trust in the xml file st=m_db->Prepare("DELETE FROM tblPeerTrust WHERE IdentityID=?;"); @@ -73,19 +118,34 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Finalize(); st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;"); - trustst=m_db->Prepare("INSERT INTO tblPeerTrust(IdentityID,TargetIdentityID,MessageTrust,TrustListTrust) VALUES(?,?,?,?);"); + trustst=m_db->Prepare("INSERT INTO tblPeerTrust(IdentityID,TargetIdentityID,MessageTrust,TrustListTrust,MessageTrustComment,TrustListTrustComment) VALUES(?,?,?,?,?,?);"); + + SQLite3DB::Statement idinsert=m_db->Prepare("INSERT INTO tblIdentity(PublicKey,DateAdded,AddedMethod) VALUES(?,?,?);"); + // loop through all trust entries in xml and add to database if we don't already know them for(long i=0; iExecuteInsert("INSERT INTO tblIdentity(PublicKey,DateAdded) VALUES('"+identity+"','"+now.Format("%Y-%m-%d %H:%M:%S")+"');",(long &)id); + if(insertcount<50 && (dayinsertcount<100 || previnsertcount==0)) + { + idinsert.Bind(0,identity); + idinsert.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")); + idinsert.Bind(2,"trust list of "+publisherid); + idinsert.Step(true); + id=idinsert.GetLastInsertRowID(); + idinsert.Reset(); + } + insertcount++; + dayinsertcount++; } else { @@ -94,17 +154,56 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Reset(); //insert trust for this identity - trustst.Bind(0,identityid); - trustst.Bind(1,id); - trustst.Bind(2,xml.GetMessageTrust(i)); - trustst.Bind(3,xml.GetTrustListTrust(i)); - trustst.Step(); - trustst.Reset(); - + if(id!=-1) + { + trustst.Bind(0,identityid); + trustst.Bind(1,id); + if(xml.GetMessageTrust(i)==-1) + { + trustst.Bind(2); + } + else + { + trustst.Bind(2,xml.GetMessageTrust(i)); + } + if(xml.GetTrustListTrust(i)==-1) + { + trustst.Bind(3); + } + else + { + trustst.Bind(3,xml.GetTrustListTrust(i)); + } + messagetrustcomment=xml.GetMessageTrustComment(i); + trustlisttrustcomment=xml.GetTrustListTrustComment(i); + // limit comments to 50 characters each + if(messagetrustcomment.size()>50) + { + messagetrustcomment.erase(50); + } + if(trustlisttrustcomment.size()>50) + { + trustlisttrustcomment.erase(50); + } + trustst.Bind(4,messagetrustcomment); + trustst.Bind(5,trustlisttrustcomment); + trustst.Step(); + trustst.Reset(); + } } + trustst.Finalize(); st.Finalize(); + if(insertcount>=50) + { + m_log->warning("TrustListRequester::HandleAllData TrustList contained more than 50 new identities : "+message["Identifier"]); + } + if(dayinsertcount>=100 && previnsertcount>0) + { + m_log->warning("TrustListRequester::HandleAllData TrustList would have inserted more than 100 new identities in the last 24 hours : "+message["Identifier"]); + } + st=m_db->Prepare("INSERT INTO tblTrustListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');"); st.Bind(0,identityid); st.Bind(1,idparts[4]); @@ -112,7 +211,9 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Step(); st.Finalize(); - m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" parsed TrustList XML file : "+message["Identifier"]); + m_db->Execute("COMMIT;"); + + m_log->debug("TrustListRequester::HandleAllData parsed TrustList XML file : "+message["Identifier"]); } else { @@ -124,7 +225,7 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Step(); st.Finalize(); - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,__FUNCTION__" error parsing TrustList XML file : "+message["Identifier"]); + m_log->error("TrustListRequester::HandleAllData error parsing TrustList XML file : "+message["Identifier"]); } // remove this identityid from request list @@ -134,15 +235,13 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) } -const bool TrustListRequester::HandleGetFailed(FCPMessage &message) +const bool TrustListRequester::HandleGetFailed(FCPv2::Message &message) { - DateTime now; SQLite3DB::Statement st; std::vector idparts; long identityid; long index; - now.SetToGMTime(); StringFunctions::Split(message["Identifier"],"|",idparts); StringFunctions::Convert(idparts[1],identityid); StringFunctions::Convert(idparts[2],index); @@ -157,7 +256,7 @@ const bool TrustListRequester::HandleGetFailed(FCPMessage &message) st.Step(); st.Finalize(); - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,__FUNCTION__" fatal error requesting "+message["Identifier"]); + m_log->error("TrustListRequester::HandleGetFailed fatal error requesting "+message["Identifier"]); } // remove this identityid from request list @@ -167,68 +266,38 @@ const bool TrustListRequester::HandleGetFailed(FCPMessage &message) } -const bool TrustListRequester::HandleMessage(FCPMessage &message) -{ - - if(message["Identifier"].find("TrustListRequester")==0) - { - if(message.GetName()=="DataFound") - { - return true; - } - - if(message.GetName()=="AllData") - { - return HandleAllData(message); - } - - if(message.GetName()=="GetFailed") - { - return HandleGetFailed(message); - } - - if(message.GetName()=="IdentifierCollision") - { - // remove one of the ids from the requesting list - long identityid=0; - std::vector idparts; - StringFunctions::Split(message["Identifier"],"|",idparts); - StringFunctions::Convert(idparts[1],identityid); - RemoveFromRequestList(identityid); - return true; - } - } - - return false; -} - void TrustListRequester::Initialize() { std::string tempval=""; - Option::instance()->Get("MaxIdentityRequests",tempval); - StringFunctions::Convert(tempval,m_maxrequests); + m_fcpuniquename="TrustListRequester"; + m_maxrequests=0; + Option option(m_db); + + option.GetInt("MaxIdentityRequests",m_maxrequests); if(m_maxrequests<1) { m_maxrequests=1; - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"Option MaxTrustListRequests is currently set at "+tempval+". It must be 1 or greater."); + m_log->error("Option MaxTrustListRequests is currently set at "+tempval+". It must be 1 or greater."); } if(m_maxrequests>100) { - m_log->WriteLog(LogFile::LOGLEVEL_WARNING,"Option MaxTrustListRequests is currently set at "+tempval+". This value might be incorrectly configured."); + m_log->warning("Option MaxTrustListRequests is currently set at "+tempval+". This value might be incorrectly configured."); } - Option::instance()->Get("MessageBase",m_messagebase); - m_tempdate.SetToGMTime(); + m_tempdate=Poco::Timestamp(); } void TrustListRequester::PopulateIDList() { - DateTime date; + Poco::DateTime date; int id; - - date.SetToGMTime(); + std::string sql; // select identities we want to query (we've seen them today and they are publishing trust list) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) - SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+date.Format("%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + sql="SELECT IdentityID FROM tblIdentity "; + sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') )"; + sql+="ORDER BY LocalTrustListTrust DESC, LastSeen;"; + + SQLite3DB::Statement st=m_db->Prepare(sql); st.Step(); m_ids.clear(); @@ -241,66 +310,10 @@ void TrustListRequester::PopulateIDList() } } -void TrustListRequester::Process() -{ - // max is the smaller of the config value or the total number of identities we will request from - long max=m_maxrequests>m_ids.size() ? m_ids.size() : m_maxrequests; - - // try to keep up to max requests going - if(m_requesting.size()::iterator i=m_ids.begin(); - while(i!=m_ids.end() && (*i).second==true) - { - i++; - } - - if(i!=m_ids.end()) - { - StartRequest((*i).first); - } - else - { - // we requested from all ids in the list, repopulate the list - PopulateIDList(); - } - } - // special case - if there were 0 identities on the list when we started then we will never get a chance to repopulate the list - // this will recheck for ids every minute - DateTime now; - now.SetToGMTime(); - if(m_tempdate<(now-(1.0/1440.0))) - { - PopulateIDList(); - m_tempdate=now; - } - -} - -void TrustListRequester::RegisterWithThread(FreenetMasterThread *thread) -{ - thread->RegisterFCPConnected(this); - thread->RegisterFCPMessageHandler(this); - thread->RegisterPeriodicProcessor(this); -} - -void TrustListRequester::RemoveFromRequestList(const long identityid) -{ - std::vector::iterator i=m_requesting.begin(); - while(i!=m_requesting.end() && (*i)!=identityid) - { - i++; - } - if(i!=m_requesting.end()) - { - m_requesting.erase(i); - } -} - -void TrustListRequester::StartRequest(const long identityid) +void TrustListRequester::StartRequest(const long &identityid) { - DateTime now; - FCPMessage message; + Poco::DateTime now; + FCPv2::Message message; std::string publickey; int index; std::string indexstr; @@ -314,10 +327,8 @@ void TrustListRequester::StartRequest(const long identityid) { st.ResultText(0,publickey); - now.SetToGMTime(); - SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblTrustListRequests WHERE Day=? AND IdentityID=?;"); - st2.Bind(0,now.Format("%Y-%m-%d")); + st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d")); st2.Bind(1,identityid); st2.Step(); @@ -336,12 +347,12 @@ void TrustListRequester::StartRequest(const long identityid) StringFunctions::Convert(identityid,identityidstr); message.SetName("ClientGet"); - message["URI"]=publickey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; - message["Identifier"]="TrustListRequester|"+identityidstr+"|"+indexstr+"|"+message["URI"]; + message["URI"]=publickey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; + message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|"+message["URI"]; message["ReturnType"]="direct"; message["MaxSize"]="1000000"; // 1 MB - m_fcp->SendMessage(message); + m_fcp->Send(message); m_requesting.push_back(identityid); }