X-Git-Url: https://git.pterodactylus.net/?a=blobdiff_plain;f=src%2Ffreenet%2Ftrustlistrequester.cpp;h=96d24df96934a6dc96dbfd1be0e8def74813081c;hb=59a5414ec47a2932a7802fcd1d98c4d80166564f;hp=77b03b19c0bc7274d21361c36a74ed20ec6b397e;hpb=ee580d19b7920904587e18d72a3465d52eab6204;p=fms.git diff --git a/src/freenet/trustlistrequester.cpp b/src/freenet/trustlistrequester.cpp index 77b03b1..96d24df 100644 --- a/src/freenet/trustlistrequester.cpp +++ b/src/freenet/trustlistrequester.cpp @@ -3,34 +3,26 @@ #include "../../include/stringfunctions.h" #include "../../include/freenet/trustlistxml.h" +#include +#include + #ifdef XMEM #include #endif -TrustListRequester::TrustListRequester() +TrustListRequester::TrustListRequester(SQLite3DB::DB *db):IIndexRequester(db) { Initialize(); } -TrustListRequester::TrustListRequester(FCPv2 *fcp):IFCPConnected(fcp) +TrustListRequester::TrustListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester(db,fcp) { Initialize(); } -void TrustListRequester::FCPConnected() -{ - m_requesting.clear(); - PopulateIDList(); -} - -void TrustListRequester::FCPDisconnected() -{ - -} - -const bool TrustListRequester::HandleAllData(FCPMessage &message) +const bool TrustListRequester::HandleAllData(FCPv2::Message &message) { - DateTime now; + Poco::DateTime now; SQLite3DB::Statement st; SQLite3DB::Statement trustst; std::vector idparts; @@ -39,32 +31,85 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) TrustListXML xml; long identityid; long index; + int insertcount=0; + int dayinsertcount=0; + int previnsertcount=0; - now.SetToGMTime(); StringFunctions::Split(message["Identifier"],"|",idparts); StringFunctions::Convert(message["DataLength"],datalength); StringFunctions::Convert(idparts[1],identityid); StringFunctions::Convert(idparts[2],index); // wait for all data to be received from connection - while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()Update(1); - } + m_fcp->WaitForBytes(1000,datalength); // if we got disconnected- return immediately - if(m_fcp->Connected()==false) + if(m_fcp->IsConnected()==false) { return false; } // receive the file - data.resize(datalength); - m_fcp->ReceiveRaw(&data[0],datalength); + m_fcp->Receive(data,datalength); + + // get count of identities added in last 24 hours + st=m_db->Prepare("SELECT COUNT(*) FROM tblIdentity WHERE DateAdded>=?;"); + now-=Poco::Timespan(1,0,0,0,0); + st.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")); + st.Step(); + if(st.RowReturned()) + { + if(st.ResultNull(0)==false) + { + st.ResultInt(0,dayinsertcount); + } + } + else + { + m_log->error("TrustListRequester::HandleAllData couldn't get count of identities added in last 24 hours"); + } + + // get count of identities added more than 24 hours ago - if 0 then we will accept more than 100 identities now + st=m_db->Prepare("SELECT COUNT(*) FROM tblIdentity WHERE DateAddederror("TrustListRequester::HandleAllData couldn't get count of identities added more than 24 hours ago"); + } + + now=Poco::DateTime(); // parse file into xml and update the database - if(xml.ParseXML(std::string(data.begin(),data.end()))==true) + if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true) { + // find the identity name and public key of the identity publishing the trust list + std::string publisherid=""; + st=m_db->Prepare("SELECT Name,PublicKey FROM tblIdentity WHERE IdentityID=?;"); + st.Bind(0,identityid); + st.Step(); + if(st.RowReturned()) + { + std::string publishername=""; + std::string publisherpublickey=""; + st.ResultText(0,publishername); + st.ResultText(1,publisherpublickey); + publisherid=publishername; + if(publisherpublickey.size()>4) + { + publisherid+=publisherpublickey.substr(3,44); + } + } + st.Finalize(); + + m_db->Execute("BEGIN;"); // drop all existing peer trust from this identity - we will rebuild it when we go through each trust in the xml file st=m_db->Prepare("DELETE FROM tblPeerTrust WHERE IdentityID=?;"); @@ -73,19 +118,34 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Finalize(); st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;"); - trustst=m_db->Prepare("INSERT INTO tblPeerTrust(IdentityID,TargetIdentityID,MessageTrust,TrustListTrust) VALUES(?,?,?,?);"); + trustst=m_db->Prepare("INSERT INTO tblPeerTrust(IdentityID,TargetIdentityID,MessageTrust,TrustListTrust,MessageTrustComment,TrustListTrustComment) VALUES(?,?,?,?,?,?);"); + + SQLite3DB::Statement idinsert=m_db->Prepare("INSERT INTO tblIdentity(PublicKey,DateAdded,AddedMethod) VALUES(?,?,?);"); + // loop through all trust entries in xml and add to database if we don't already know them for(long i=0; iExecuteInsert("INSERT INTO tblIdentity(PublicKey,DateAdded) VALUES('"+identity+"','"+now.Format("%Y-%m-%d %H:%M:%S")+"');",(long &)id); + if(insertcount<50 && (dayinsertcount<100 || previnsertcount==0)) + { + idinsert.Bind(0,identity); + idinsert.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")); + idinsert.Bind(2,"trust list of "+publisherid); + idinsert.Step(true); + id=idinsert.GetLastInsertRowID(); + idinsert.Reset(); + } + insertcount++; + dayinsertcount++; } else { @@ -94,31 +154,56 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Reset(); //insert trust for this identity - trustst.Bind(0,identityid); - trustst.Bind(1,id); - if(xml.GetMessageTrust(i)==-1) + if(id!=-1) { - trustst.Bind(2); + trustst.Bind(0,identityid); + trustst.Bind(1,id); + if(xml.GetMessageTrust(i)==-1) + { + trustst.Bind(2); + } + else + { + trustst.Bind(2,xml.GetMessageTrust(i)); + } + if(xml.GetTrustListTrust(i)==-1) + { + trustst.Bind(3); + } + else + { + trustst.Bind(3,xml.GetTrustListTrust(i)); + } + messagetrustcomment=xml.GetMessageTrustComment(i); + trustlisttrustcomment=xml.GetTrustListTrustComment(i); + // limit comments to 50 characters each + if(messagetrustcomment.size()>50) + { + messagetrustcomment.erase(50); + } + if(trustlisttrustcomment.size()>50) + { + trustlisttrustcomment.erase(50); + } + trustst.Bind(4,messagetrustcomment); + trustst.Bind(5,trustlisttrustcomment); + trustst.Step(); + trustst.Reset(); } - else - { - trustst.Bind(2,xml.GetMessageTrust(i)); - } - if(xml.GetTrustListTrust(i)==-1) - { - trustst.Bind(3); - } - else - { - trustst.Bind(3,xml.GetTrustListTrust(i)); - } - trustst.Step(); - trustst.Reset(); - } + trustst.Finalize(); st.Finalize(); + if(insertcount>=50) + { + m_log->warning("TrustListRequester::HandleAllData TrustList contained more than 50 new identities : "+message["Identifier"]); + } + if(dayinsertcount>=100 && previnsertcount>0) + { + m_log->warning("TrustListRequester::HandleAllData TrustList would have inserted more than 100 new identities in the last 24 hours : "+message["Identifier"]); + } + st=m_db->Prepare("INSERT INTO tblTrustListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');"); st.Bind(0,identityid); st.Bind(1,idparts[4]); @@ -126,7 +211,9 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Step(); st.Finalize(); - m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,"TrustListRequester::HandleAllData parsed TrustList XML file : "+message["Identifier"]); + m_db->Execute("COMMIT;"); + + m_log->debug("TrustListRequester::HandleAllData parsed TrustList XML file : "+message["Identifier"]); } else { @@ -138,7 +225,7 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) st.Step(); st.Finalize(); - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"TrustListRequester::HandleAllData error parsing TrustList XML file : "+message["Identifier"]); + m_log->error("TrustListRequester::HandleAllData error parsing TrustList XML file : "+message["Identifier"]); } // remove this identityid from request list @@ -148,15 +235,13 @@ const bool TrustListRequester::HandleAllData(FCPMessage &message) } -const bool TrustListRequester::HandleGetFailed(FCPMessage &message) +const bool TrustListRequester::HandleGetFailed(FCPv2::Message &message) { - DateTime now; SQLite3DB::Statement st; std::vector idparts; long identityid; long index; - now.SetToGMTime(); StringFunctions::Split(message["Identifier"],"|",idparts); StringFunctions::Convert(idparts[1],identityid); StringFunctions::Convert(idparts[2],index); @@ -171,7 +256,7 @@ const bool TrustListRequester::HandleGetFailed(FCPMessage &message) st.Step(); st.Finalize(); - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"TrustListRequester::HandleGetFailed fatal error requesting "+message["Identifier"]); + m_log->error("TrustListRequester::HandleGetFailed fatal error requesting "+message["Identifier"]); } // remove this identityid from request list @@ -181,70 +266,35 @@ const bool TrustListRequester::HandleGetFailed(FCPMessage &message) } -const bool TrustListRequester::HandleMessage(FCPMessage &message) -{ - - if(message["Identifier"].find("TrustListRequester")==0) - { - if(message.GetName()=="DataFound") - { - return true; - } - - if(message.GetName()=="AllData") - { - return HandleAllData(message); - } - - if(message.GetName()=="GetFailed") - { - return HandleGetFailed(message); - } - - if(message.GetName()=="IdentifierCollision") - { - // remove one of the ids from the requesting list - long identityid=0; - std::vector idparts; - StringFunctions::Split(message["Identifier"],"|",idparts); - StringFunctions::Convert(idparts[1],identityid); - RemoveFromRequestList(identityid); - return true; - } - } - - return false; -} - void TrustListRequester::Initialize() { std::string tempval=""; - Option::Instance()->Get("MaxIdentityRequests",tempval); - StringFunctions::Convert(tempval,m_maxrequests); + m_fcpuniquename="TrustListRequester"; + m_maxrequests=0; + Option option(m_db); + + option.GetInt("MaxIdentityRequests",m_maxrequests); if(m_maxrequests<1) { m_maxrequests=1; - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"Option MaxTrustListRequests is currently set at "+tempval+". It must be 1 or greater."); + m_log->error("Option MaxTrustListRequests is currently set at "+tempval+". It must be 1 or greater."); } if(m_maxrequests>100) { - m_log->WriteLog(LogFile::LOGLEVEL_WARNING,"Option MaxTrustListRequests is currently set at "+tempval+". This value might be incorrectly configured."); + m_log->warning("Option MaxTrustListRequests is currently set at "+tempval+". This value might be incorrectly configured."); } - Option::Instance()->Get("MessageBase",m_messagebase); - m_tempdate.SetToGMTime(); + m_tempdate=Poco::Timestamp(); } void TrustListRequester::PopulateIDList() { - DateTime date; + Poco::DateTime date; int id; std::string sql; - date.SetToGMTime(); - // select identities we want to query (we've seen them today and they are publishing trust list) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) sql="SELECT IdentityID FROM tblIdentity "; - sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+date.Format("%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') )"; + sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') )"; sql+="ORDER BY LocalTrustListTrust DESC, LastSeen;"; SQLite3DB::Statement st=m_db->Prepare(sql); @@ -260,66 +310,10 @@ void TrustListRequester::PopulateIDList() } } -void TrustListRequester::Process() -{ - // max is the smaller of the config value or the total number of identities we will request from - long max=m_maxrequests>m_ids.size() ? m_ids.size() : m_maxrequests; - - // try to keep up to max requests going - if(m_requesting.size()::iterator i=m_ids.begin(); - while(i!=m_ids.end() && (*i).second==true) - { - i++; - } - - if(i!=m_ids.end()) - { - StartRequest((*i).first); - } - else - { - // we requested from all ids in the list, repopulate the list - PopulateIDList(); - } - } - // special case - if there were 0 identities on the list when we started then we will never get a chance to repopulate the list - // this will recheck for ids every minute - DateTime now; - now.SetToGMTime(); - if(m_ids.size()==0 && m_tempdate<(now-(1.0/1440.0))) - { - PopulateIDList(); - m_tempdate=now; - } - -} - -void TrustListRequester::RegisterWithThread(FreenetMasterThread *thread) -{ - thread->RegisterFCPConnected(this); - thread->RegisterFCPMessageHandler(this); - thread->RegisterPeriodicProcessor(this); -} - -void TrustListRequester::RemoveFromRequestList(const long identityid) -{ - std::vector::iterator i=m_requesting.begin(); - while(i!=m_requesting.end() && (*i)!=identityid) - { - i++; - } - if(i!=m_requesting.end()) - { - m_requesting.erase(i); - } -} - -void TrustListRequester::StartRequest(const long identityid) +void TrustListRequester::StartRequest(const long &identityid) { - DateTime now; - FCPMessage message; + Poco::DateTime now; + FCPv2::Message message; std::string publickey; int index; std::string indexstr; @@ -333,10 +327,8 @@ void TrustListRequester::StartRequest(const long identityid) { st.ResultText(0,publickey); - now.SetToGMTime(); - SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblTrustListRequests WHERE Day=? AND IdentityID=?;"); - st2.Bind(0,now.Format("%Y-%m-%d")); + st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d")); st2.Bind(1,identityid); st2.Step(); @@ -355,12 +347,12 @@ void TrustListRequester::StartRequest(const long identityid) StringFunctions::Convert(identityid,identityidstr); message.SetName("ClientGet"); - message["URI"]=publickey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; - message["Identifier"]="TrustListRequester|"+identityidstr+"|"+indexstr+"|"+message["URI"]; + message["URI"]=publickey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; + message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|"+message["URI"]; message["ReturnType"]="direct"; message["MaxSize"]="1000000"; // 1 MB - m_fcp->SendMessage(message); + m_fcp->Send(message); m_requesting.push_back(identityid); }