X-Git-Url: https://git.pterodactylus.net/?a=blobdiff_plain;f=src%2Ffreenet%2Ftrustlistinserter.cpp;h=10bfbceca8f65541bd8f167e94dac6738adcdbfe;hb=59a5414ec47a2932a7802fcd1d98c4d80166564f;hp=c5cc7df36fda8ddb9ca5ee38096811a2a31f2ef3;hpb=1b0c3b7f86935a772aad271bad4f3d1f37243c2d;p=fms.git diff --git a/src/freenet/trustlistinserter.cpp b/src/freenet/trustlistinserter.cpp index c5cc7df..10bfbce 100644 --- a/src/freenet/trustlistinserter.cpp +++ b/src/freenet/trustlistinserter.cpp @@ -3,26 +3,35 @@ #include "../../include/freenet/trustlistxml.h" #include "../../include/stringfunctions.h" +#include +#include +#include + #ifdef XMEM #include #endif -TrustListInserter::TrustListInserter() +TrustListInserter::TrustListInserter(SQLite3DB::DB *db):IDatabase(db) { Initialize(); } -TrustListInserter::TrustListInserter(FCPv2 *fcp):IFCPConnected(fcp) +TrustListInserter::TrustListInserter(SQLite3DB::DB *db, FCPv2::Connection *fcp):IDatabase(db),IFCPConnected(fcp) { Initialize(); } void TrustListInserter::CheckForNeededInsert() { - DateTime date; - date.SetToGMTime(); - date.Add(0,0,-1); - SQLite3DB::Recordset rs=m_db->Query("SELECT LocalIdentityID, PrivateKey FROM tblLocalIdentity WHERE PrivateKey IS NOT NULL AND PrivateKey <> '' AND PublishTrustList='true' AND InsertingTrustList='false' AND (LastInsertedTrustList<='"+date.Format("%Y-%m-%d %H:%M:%S")+"' OR LastInsertedTrustList IS NULL);"); + Poco::DateTime date; + int currentday=date.day(); + date-=Poco::Timespan(0,6,0,0,0); + // insert trust lists every 6 hours - if 6 hours ago was different day then set to midnight of current day to insert list today ASAP + if(currentday!=date.day()) + { + date.assign(date.year(),date.month(),currentday,0,0,0); + } + SQLite3DB::Recordset rs=m_db->Query("SELECT LocalIdentityID, PrivateKey FROM tblLocalIdentity WHERE PrivateKey IS NOT NULL AND PrivateKey <> '' AND PublishTrustList='true' AND InsertingTrustList='false' AND (LastInsertedTrustList<='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S")+"' OR LastInsertedTrustList IS NULL);"); if(rs.Empty()==false) { @@ -40,16 +49,15 @@ void TrustListInserter::FCPDisconnected() } -const bool TrustListInserter::HandleMessage(FCPMessage &message) +const bool TrustListInserter::HandleMessage(FCPv2::Message &message) { if(message["Identifier"].find("TrustListInserter")==0) { - DateTime now; + Poco::DateTime now; std::vector idparts; - now.SetToGMTime(); StringFunctions::Split(message["Identifier"],"|",idparts); // no action for URIGenerated @@ -66,23 +74,30 @@ const bool TrustListInserter::HandleMessage(FCPMessage &message) if(message.GetName()=="PutSuccessful") { - m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false', LastInsertedTrustList='"+now.Format("%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";"); - m_db->Execute("INSERT INTO tblTrustListInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); - m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" inserted TrustList xml"); + // non USK + if(idparts[0]=="TrustListInserter") + { + m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false', LastInsertedTrustList='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";"); + m_db->Execute("INSERT INTO tblTrustListInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + m_log->debug("TrustListInserter::HandleMessage inserted TrustList xml"); + } return true; } - if(message.GetName()=="PutFailed") + if(message.GetName()=="PutFailed" && idparts[0]=="TrustListInserter") { - m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false' WHERE LocalIdentityID="+idparts[1]+";"); - m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" failure inserting TrustList xml. Code="+message["Code"]+" Description="+message["CodeDescription"]); - - // if code 9 (collision), then insert index into inserted table - if(message["Code"]=="9") + // non USK + if(idparts[0]=="TrustListInserter") { - m_db->Execute("INSERT INTO tblTrustListInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); - } + m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false' WHERE LocalIdentityID="+idparts[1]+";"); + m_log->debug("TrustListInserter::HandleMessage failure inserting TrustList xml. Code="+message["Code"]+" Description="+message["CodeDescription"]); + // if code 9 (collision), then insert index into inserted table + if(message["Code"]=="9") + { + m_db->Execute("INSERT INTO tblTrustListInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + } + } return true; } @@ -93,17 +108,17 @@ const bool TrustListInserter::HandleMessage(FCPMessage &message) void TrustListInserter::Initialize() { - Option::instance()->Get("MessageBase",m_messagebase); - m_lastchecked.SetToGMTime(); + Option option(m_db); + option.Get("MessageBase",m_messagebase); + m_lastchecked=Poco::Timestamp(); } void TrustListInserter::Process() { - DateTime now; - now.SetToGMTime(); + Poco::DateTime now; // check every minute - if(m_lastchecked<=(now-(1.0/1440.0))) + if(m_lastchecked<=(now-Poco::Timespan(0,0,1,0,0))) { CheckForNeededInsert(); m_lastchecked=now; @@ -119,36 +134,110 @@ void TrustListInserter::RegisterWithThread(FreenetMasterThread *thread) void TrustListInserter::StartInsert(const long localidentityid, const std::string &privatekey) { - FCPMessage message; + FCPv2::Message message; TrustListXML xml; std::string data; std::string datasizestr; std::string publickey; int messagetrust; int trustlisttrust; - DateTime now; + Poco::DateTime now,date,dateminus30,tempdate; int index; std::string indexstr; std::string localidentityidstr; + std::string messagetrustcomment=""; + std::string trustlisttrustcomment=""; + int identityid=-1; + int count=0; + bool add=false; + std::string dateadded=""; - now.SetToGMTime(); - - // build the xml file - SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey, LocalMessageTrust, LocalTrustListTrust FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey<>'';"); + dateminus30-=Poco::Timespan(30,0,0,0,0); + + // insert all identities not in trust list already + m_db->Execute("INSERT INTO tblIdentityTrust(LocalIdentityID,IdentityID) SELECT LocalIdentityID,IdentityID FROM tblLocalIdentity,tblIdentity WHERE LocalIdentityID || '_' || IdentityID NOT IN (SELECT LocalIdentityID || '_' || IdentityID FROM tblIdentityTrust);"); + + // select statement for last message date for an identity + SQLite3DB::Statement countst=m_db->Prepare("SELECT COUNT(*) FROM tblMessage WHERE IdentityID=? AND MessageDate>=?;"); + + // build the xml file - we only want to add identities that we recently saw, otherwise we could be inserting a ton of identities + date-=Poco::Timespan(15,0,0,0,0); // identities seen in last 15 days - the maintenance page lets us delete identities not seen in 20 days, so this gives us a window where the identity won't be deleted and then found in a trust list and readded immediately + //SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey, LocalMessageTrust, LocalTrustListTrust, MessageTrustComment, TrustListTrustComment FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey<>'' AND LastSeen>=?;"); + // we want to order by public key so we can't do identity correllation based on the sequence of identities in the list. + SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey, tblIdentityTrust.LocalMessageTrust, tblIdentityTrust.LocalTrustListTrust, tblIdentityTrust.MessageTrustComment, tblIdentityTrust.TrustListTrustComment, tblIdentity.IdentityID, tblIdentity.DateAdded FROM tblIdentity INNER JOIN tblIdentityTrust ON tblIdentity.IdentityID=tblIdentityTrust.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey<>'' AND LastSeen>=? AND tblIdentityTrust.LocalIdentityID=? ORDER BY PublicKey;"); + st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d")); + st.Bind(1,localidentityid); st.Step(); while(st.RowReturned()) { st.ResultText(0,publickey); - st.ResultInt(1,messagetrust); - st.ResultInt(2,trustlisttrust); - xml.AddTrust(publickey,messagetrust,trustlisttrust); + if(st.ResultNull(1)==false) + { + st.ResultInt(1,messagetrust); + } + else + { + messagetrust=-1; + } + if(st.ResultNull(2)==false) + { + st.ResultInt(2,trustlisttrust); + } + else + { + trustlisttrust=-1; + } + st.ResultText(3,messagetrustcomment); + st.ResultText(4,trustlisttrustcomment); + identityid=-1; + st.ResultInt(5,identityid); + dateadded=""; + st.ResultText(6,dateadded); + + add=false; + + // add the identity to the trust list if they have posted a message in the last 30 days + countst.Bind(0,identityid); + countst.Bind(1,Poco::DateTimeFormatter::format(dateminus30,"%Y-%m-%d")); + countst.Step(); + if(countst.RowReturned()) + { + count=0; + countst.ResultInt(0,count); + if(count>0) + { + add=true; + } + } + countst.Reset(); + + // no messages in last 30 days - add the identity if we learned about them less than 5 days ago + if(add==false && dateadded!="") + { + int tzdiff=0; + if(Poco::DateTimeParser::tryParse(dateadded,tempdate,tzdiff)==false) + { + tempdate=Poco::Timestamp(); + m_log->fatal("TrustListInserter::StartInsert could not parse date "+dateadded); + } + if(tempdate>=(now-Poco::Timespan(5,0,0,0,0))) + { + add=true; + } + } + + if(add==true) + { + xml.AddTrust(publickey,messagetrust,trustlisttrust,messagetrustcomment,trustlisttrustcomment); + } + st.Step(); } // get next insert index st=m_db->Prepare("SELECT MAX(InsertIndex) FROM tblTrustListInserts WHERE LocalIdentityID=? AND Day=?;"); st.Bind(0,localidentityid); - st.Bind(1,now.Format("%Y-%m-%d")); + st.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d")); st.Step(); index=0; @@ -165,13 +254,25 @@ void TrustListInserter::StartInsert(const long localidentityid, const std::strin StringFunctions::Convert(data.size(),datasizestr); message.SetName("ClientPut"); - message["URI"]=privatekey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; + message["URI"]=privatekey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; message["Identifier"]="TrustListInserter|"+localidentityidstr+"|"+indexstr+"|"+message["URI"]; message["UploadFrom"]="direct"; message["DataLength"]=datasizestr; - m_fcp->SendMessage(message); - m_fcp->SendRaw(data.c_str(),data.size()); + m_fcp->Send(message); + m_fcp->Send(std::vector(data.begin(),data.end())); + + // insert to USK + message.Clear(); + message.SetName("ClientPutComplexDir"); + message["URI"]="USK"+privatekey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|TrustList/0/"; + message["Identifier"]="TrustListInserterUSK|"+message["URI"]; + message["DefaultName"]="TrustList.xml"; + message["Files.0.Name"]="TrustList.xml"; + message["Files.0.UplaodFrom"]="direct"; + message["Files.0.DataLength"]=datasizestr; + m_fcp->Send(message); + m_fcp->Send(std::vector(data.begin(),data.end())); m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='true' WHERE LocalIdentityID="+localidentityidstr+";"); -} \ No newline at end of file +}