From 1b0c3b7f86935a772aad271bad4f3d1f37243c2d Mon Sep 17 00:00:00 2001 From: SomeDude Date: Thu, 17 Jan 2008 17:58:00 +0100 Subject: [PATCH] version 0.0.2 --- admin/introduce.php | 2 +- admin/introductionstatus.php | 42 ++++ admin/linkbar.php | 1 + include/freenet/trustlistinserter.h | 37 ++++ include/freenet/trustlistrequester.h | 43 ++++ include/freenet/trustlistxml.h | 42 ++++ include/identitytestglobal.h | 2 +- src/datetime.cpp | 14 +- src/freenet/freenetmasterthread.cpp | 7 +- src/freenet/identityinserter.cpp | 35 ++- src/freenet/identityrequester.cpp | 20 +- src/freenet/introductionpuzzleinserter.cpp | 6 + src/freenet/trustlistinserter.cpp | 177 ++++++++++++++++ src/freenet/trustlistrequester.cpp | 329 +++++++++++++++++++++++++++++ src/freenet/trustlistxml.cpp | 148 +++++++++++++ src/identitytestglobal.cpp | 80 ++++++- src/identitytestmain.cpp | 1 - 17 files changed, 961 insertions(+), 25 deletions(-) create mode 100644 admin/introductionstatus.php create mode 100644 include/freenet/trustlistinserter.h create mode 100644 include/freenet/trustlistrequester.h create mode 100644 include/freenet/trustlistxml.h create mode 100644 src/freenet/trustlistinserter.cpp create mode 100644 src/freenet/trustlistrequester.cpp create mode 100644 src/freenet/trustlistxml.cpp diff --git a/admin/introduce.php b/admin/introduce.php index 29fe5e7..f2103a8 100644 --- a/admin/introduce.php +++ b/admin/introduce.php @@ -53,7 +53,7 @@ function content() print "
Type answers for a few puzzles and submit
"; - $st=$db->prepare("SELECT UUID,Day FROM tblIntroductionPuzzleRequests WHERE UUID NOT IN (SELECT UUID FROM tblIdentityIntroductionInserts) AND Day>='".gmdate('Y-m-d',strtotime('-1 day'))."' AND Found='true';"); + $st=$db->prepare("SELECT UUID,Day FROM tblIntroductionPuzzleRequests WHERE UUID NOT IN (SELECT UUID FROM tblIdentityIntroductionInserts) AND Day>='".gmdate('Y-m-d',strtotime('-1 day'))."' AND Found='true';"); $st->execute(); while($record=$st->fetch()) diff --git a/admin/introductionstatus.php b/admin/introductionstatus.php new file mode 100644 index 0000000..b4b5492 --- /dev/null +++ b/admin/introductionstatus.php @@ -0,0 +1,42 @@ +prepare("SELECT tblLocalIdentity.Name, COUNT(Inserted) FROM tblLocalIdentity LEFT JOIN tblIdentityIntroductionInserts ON tblLocalIdentity.LocalIdentityID=tblIdentityIntroductionInserts.LocalIdentityID WHERE (Inserted='true' OR Inserted IS NULL) GROUP BY tblLocalIdentity.LocalIdentityID;"); + $st->execute(); + + ?> +

Introduction Status

+ + + + + + fetch()) + { + ?> + + + + + +
IdentitySuccessful introduction inserts
+ + + +
+ \ No newline at end of file diff --git a/admin/linkbar.php b/admin/linkbar.php index 8876cf0..6f39f5e 100644 --- a/admin/linkbar.php +++ b/admin/linkbar.php @@ -10,6 +10,7 @@ function linkbar()
  • Options
  • Create Identity
  • Introduce Identity
    • +
  • Introduction Status
  • Manually Add Peer
    • diff --git a/include/freenet/trustlistinserter.h b/include/freenet/trustlistinserter.h new file mode 100644 index 0000000..ca599dd --- /dev/null +++ b/include/freenet/trustlistinserter.h @@ -0,0 +1,37 @@ +#ifndef _trustlistinserter_ +#define _trustlistinserter_ + +#include "../idatabase.h" +#include "../ilogger.h" +#include "../datetime.h" +#include "ifreenetregistrable.h" +#include "ifcpconnected.h" +#include "ifcpmessagehandler.h" +#include "iperiodicprocessor.h" + +class TrustListInserter:public IFreenetRegistrable,public IFCPConnected,public IFCPMessageHandler,public IPeriodicProcessor,public IDatabase,public ILogger +{ +public: + TrustListInserter(); + TrustListInserter(FCPv2 *fcp); + + void FCPConnected(); + void FCPDisconnected(); + + const bool HandleMessage(FCPMessage &message); + + void Process(); + + void RegisterWithThread(FreenetMasterThread *thread); + +private: + void Initialize(); + void CheckForNeededInsert(); + void StartInsert(const long localidentityid, const std::string &privatekey); + + std::string m_messagebase; + DateTime m_lastchecked; + +}; + +#endif // _trustlistinserter_ diff --git a/include/freenet/trustlistrequester.h b/include/freenet/trustlistrequester.h new file mode 100644 index 0000000..721b9cb --- /dev/null +++ b/include/freenet/trustlistrequester.h @@ -0,0 +1,43 @@ +#ifndef _trustlistrequester_ +#define _trustlistrequester_ + +#include "../idatabase.h" +#include "../ilogger.h" +#include "../datetime.h" +#include "ifreenetregistrable.h" +#include "ifcpconnected.h" +#include "ifcpmessagehandler.h" +#include "iperiodicprocessor.h" + +class TrustListRequester:public IFreenetRegistrable,public IFCPConnected,public IFCPMessageHandler,public IPeriodicProcessor,public IDatabase,public ILogger +{ +public: + TrustListRequester(); + TrustListRequester(FCPv2 *fcp); + + void FCPDisconnected(); + void FCPConnected(); + + const bool HandleMessage(FCPMessage &message); + + void Process(); + + void RegisterWithThread(FreenetMasterThread *thread); + +private: + void Initialize(); + void PopulateIDList(); // clear and re-populate m_ids with identities we want to query + void StartRequest(const long identityid); + const bool HandleAllData(FCPMessage &message); + const bool HandleGetFailed(FCPMessage &message); + void RemoveFromRequestList(const long identityid); + + DateTime m_tempdate; + std::string m_messagebase; + long m_maxrequests; + std::vector m_requesting; // list of ids we are currently requesting from + std::map m_ids; // map of all ids we know and whether we have requested file from them yet + +}; + +#endif // _trustlistrequester_ diff --git a/include/freenet/trustlistxml.h b/include/freenet/trustlistxml.h new file mode 100644 index 0000000..bd66c02 --- /dev/null +++ b/include/freenet/trustlistxml.h @@ -0,0 +1,42 @@ +#ifndef _trustlistxml_ +#define _trustlistxml_ + +#include "../ifmsxmldocument.h" +#include "../ilogger.h" + +#include + +class TrustListXML:public IFMSXMLDocument,public ILogger +{ +public: + TrustListXML(); + + std::string GetXML(); + + const bool ParseXML(const std::string &xml); + + void ClearTrust() { m_trust.clear(); } + + void AddTrust(const std::string &identity, const long messagetrust, const long trustlisttrust); + + const long TrustCount() { return m_trust.size(); } + std::string GetIdentity(const long index); + long GetMessageTrust(const long index); + long GetTrustListTrust(const long index); + +private: + struct trust + { + trust(const std::string &identity, const long messagetrust, const long trustlisttrust):m_identity(identity),m_messagetrust(messagetrust),m_trustlisttrust(trustlisttrust) {} + std::string m_identity; + long m_messagetrust; + long m_trustlisttrust; + }; + + void Initialize(); + + std::vector m_trust; + +}; + +#endif // _trustlistxml_ diff --git a/include/identitytestglobal.h b/include/identitytestglobal.h index eff73b9..eb5be4d 100644 --- a/include/identitytestglobal.h +++ b/include/identitytestglobal.h @@ -4,7 +4,7 @@ #include #include -#define FMS_VERSION "0.0.1" +#define FMS_VERSION "0.0.2" // opens database and creates tables and initial inserts if necessary void SetupDB(); diff --git a/src/datetime.cpp b/src/datetime.cpp index 7d3d4af..9e1a7f8 100644 --- a/src/datetime.cpp +++ b/src/datetime.cpp @@ -36,19 +36,11 @@ void DateTime::Add(const int seconds, const int minutes, const int hours, const std::string DateTime::Format(const std::string &formatstring) const { - std::string returnval=""; - char *str=new char[512]; - memset(str,0,512); + std::vector str(256,0); - strftime(str,511,formatstring.c_str(),&m_tm); + size_t len=strftime(&str[0],str.size()-1,formatstring.c_str(),&m_tm); - if(str) - { - returnval=str; - delete [] str; - } - - return returnval; + return std::string(str.begin(),str.begin()+len); } void DateTime::Normalize() diff --git a/src/freenet/freenetmasterthread.cpp b/src/freenet/freenetmasterthread.cpp index 01f03a0..e0db51c 100644 --- a/src/freenet/freenetmasterthread.cpp +++ b/src/freenet/freenetmasterthread.cpp @@ -1,5 +1,6 @@ #include "../../include/freenet/freenetmasterthread.h" #include "../../include/option.h" +#include "../../include/uuidgenerator.h" #include "../../include/stringfunctions.h" #include "../../include/freenet/unkeyedidcreator.h" #include "../../include/freenet/identityinserter.h" @@ -9,6 +10,7 @@ #include "../../include/freenet/introductionpuzzlerequester.h" #include "../../include/freenet/introductionpuzzleremover.h" #include "../../include/freenet/identityintroductioninserter.h" +#include "../../include/freenet/trustlistinserter.h" #include @@ -63,8 +65,10 @@ const bool FreenetMasterThread::FCPConnect() if(m_fcp.Connect(m_fcphost.c_str(),m_fcpport)==true) { + UUIDGenerator uuid; + std::string clientname="FMSClient-"+uuid.Generate(); // send ClientHello message to node - m_fcp.SendMessage("ClientHello",2,"Name","FMSClient","ExpectedVersion","2.0"); + m_fcp.SendMessage("ClientHello",2,"Name",clientname.c_str(),"ExpectedVersion","2.0"); m_log->WriteLog(LogFile::LOGLEVEL_INFO,__FUNCTION__" connected to node"); @@ -228,6 +232,7 @@ void FreenetMasterThread::Setup() m_registrables.push_back(new IntroductionPuzzleRequester(&m_fcp)); m_registrables.push_back(new IntroductionPuzzleRemover()); m_registrables.push_back(new IdentityIntroductionInserter(&m_fcp)); + m_registrables.push_back(new TrustListInserter(&m_fcp)); for(std::vector::iterator i=m_registrables.begin(); i!=m_registrables.end(); i++) { diff --git a/src/freenet/identityinserter.cpp b/src/freenet/identityinserter.cpp index 33a8085..028f495 100644 --- a/src/freenet/identityinserter.cpp +++ b/src/freenet/identityinserter.cpp @@ -17,18 +17,21 @@ IdentityInserter::IdentityInserter(FCPv2 *fcp):IFCPConnected(fcp) Initialize(); } -void IdentityInserter::FCPConnected() -{ - m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false';"); -} - void IdentityInserter::CheckForNeededInsert() { + DateTime now; DateTime date; + now.SetToGMTime(); date.SetToGMTime(); // set date to 1 hour back date.Add(0,0,-1); + // Because of importance of Identity.xml, if we are now at the next day we immediately want to insert identities so change the date back to now + if(date.GetDay()!=now.GetDay()) + { + date=now; + } + SQLite3DB::Recordset rs=m_db->Query("SELECT LocalIdentityID FROM tblLocalIdentity WHERE PrivateKey IS NOT NULL AND PrivateKey <> '' AND InsertingIdentity='false' AND (LastInsertedIdentity<'"+date.Format("%Y-%m-%d %H:%M:%S")+"' OR LastInsertedIdentity IS NULL) ORDER BY LastInsertedIdentity;"); if(rs.Empty()==false) @@ -38,6 +41,12 @@ void IdentityInserter::CheckForNeededInsert() } +void IdentityInserter::FCPConnected() +{ + m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false';"); +} + + void IdentityInserter::FCPDisconnected() { @@ -127,7 +136,7 @@ void IdentityInserter::StartInsert(const long localidentityid) StringFunctions::Convert(localidentityid,idstring); date.SetToGMTime(); - SQLite3DB::Recordset rs=m_db->Query("SELECT Name,PrivateKey,SingleUse FROM tblLocalIdentity WHERE LocalIdentityID="+idstring+";"); + SQLite3DB::Recordset rs=m_db->Query("SELECT Name,PrivateKey,SingleUse,PublishTrustList,PublishBoardList FROM tblLocalIdentity WHERE LocalIdentityID="+idstring+";"); if(rs.Empty()==false) { @@ -141,6 +150,8 @@ void IdentityInserter::StartInsert(const long localidentityid) long index=0; std::string indexstr; std::string singleuse="false"; + std::string publishtrustlist="false"; + std::string publishboardlist="false"; now.SetToGMTime(); @@ -176,6 +187,18 @@ void IdentityInserter::StartInsert(const long localidentityid) } singleuse=="true" ? idxml.SetSingleUse(true) : idxml.SetSingleUse(false); + if(rs.GetField(3)) + { + publishtrustlist=rs.GetField(3); + } + publishtrustlist=="true" ? idxml.SetPublishTrustList(true) : idxml.SetPublishTrustList(false); + + if(rs.GetField(4)) + { + publishboardlist=rs.GetField(3); + } + publishboardlist=="true" ? idxml.SetPublishBoardList(true) : idxml.SetPublishBoardList(false); + data=idxml.GetXML(); StringFunctions::Convert(data.size(),datasizestr); diff --git a/src/freenet/identityrequester.cpp b/src/freenet/identityrequester.cpp index 056900d..229cc7b 100644 --- a/src/freenet/identityrequester.cpp +++ b/src/freenet/identityrequester.cpp @@ -66,7 +66,7 @@ const bool IdentityRequester::HandleAllData(FCPMessage &message) if(xml.ParseXML(std::string(data.begin(),data.end()))==true) { - st=m_db->Prepare("UPDATE tblIdentity SET Name=?, SingleUse=?, LastSeen=? WHERE IdentityID=?"); + st=m_db->Prepare("UPDATE tblIdentity SET Name=?, SingleUse=?, LastSeen=?, PublishTrustList=?, PublishBoardList=? WHERE IdentityID=?"); st.Bind(0,xml.GetName()); if(xml.GetSingleUse()==true) { @@ -77,7 +77,23 @@ const bool IdentityRequester::HandleAllData(FCPMessage &message) st.Bind(1,"false"); } st.Bind(2,now.Format("%Y-%m-%d %H:%M:%S")); - st.Bind(3,identityid); + if(xml.GetPublishTrustList()==true) + { + st.Bind(3,"true"); + } + else + { + st.Bind(3,"false"); + } + if(xml.GetPublishBoardList()==true) + { + st.Bind(4,"true"); + } + else + { + st.Bind(4,"false"); + } + st.Bind(5,identityid); st.Step(); st.Finalize(); diff --git a/src/freenet/introductionpuzzleinserter.cpp b/src/freenet/introductionpuzzleinserter.cpp index f87e2bd..ea78ace 100644 --- a/src/freenet/introductionpuzzleinserter.cpp +++ b/src/freenet/introductionpuzzleinserter.cpp @@ -112,6 +112,12 @@ const bool IntroductionPuzzleInserter::HandlePutFailed(FCPMessage &message) st.Step(); st.Finalize(); + // if fatal error or collision - mark index + if(message["Fatal"]=="true" || message["Code"]=="9") + { + m_db->Execute("UPDATE tblIntroductionPuzzleInserts SET Day='"+idparts[5]+"', InsertIndex="+idparts[2]+", FoundSolution='true' WHERE UUID='"+idparts[3]+"';"); + } + m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" failed to insert puzzle "+idparts[3]); return true; diff --git a/src/freenet/trustlistinserter.cpp b/src/freenet/trustlistinserter.cpp new file mode 100644 index 0000000..c5cc7df --- /dev/null +++ b/src/freenet/trustlistinserter.cpp @@ -0,0 +1,177 @@ +#include "../../include/freenet/trustlistinserter.h" +#include "../../include/option.h" +#include "../../include/freenet/trustlistxml.h" +#include "../../include/stringfunctions.h" + +#ifdef XMEM + #include +#endif + +TrustListInserter::TrustListInserter() +{ + Initialize(); +} + +TrustListInserter::TrustListInserter(FCPv2 *fcp):IFCPConnected(fcp) +{ + Initialize(); +} + +void TrustListInserter::CheckForNeededInsert() +{ + DateTime date; + date.SetToGMTime(); + date.Add(0,0,-1); + SQLite3DB::Recordset rs=m_db->Query("SELECT LocalIdentityID, PrivateKey FROM tblLocalIdentity WHERE PrivateKey IS NOT NULL AND PrivateKey <> '' AND PublishTrustList='true' AND InsertingTrustList='false' AND (LastInsertedTrustList<='"+date.Format("%Y-%m-%d %H:%M:%S")+"' OR LastInsertedTrustList IS NULL);"); + + if(rs.Empty()==false) + { + StartInsert(rs.GetInt(0),rs.GetField(1)); + } +} + +void TrustListInserter::FCPConnected() +{ + m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false';"); +} + +void TrustListInserter::FCPDisconnected() +{ + +} + +const bool TrustListInserter::HandleMessage(FCPMessage &message) +{ + + if(message["Identifier"].find("TrustListInserter")==0) + { + + DateTime now; + std::vector idparts; + + now.SetToGMTime(); + StringFunctions::Split(message["Identifier"],"|",idparts); + + // no action for URIGenerated + if(message.GetName()=="URIGenerated") + { + return true; + } + + // no action for IdentifierCollision + if(message.GetName()=="IdentifierCollision") + { + return true; + } + + if(message.GetName()=="PutSuccessful") + { + m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false', LastInsertedTrustList='"+now.Format("%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";"); + m_db->Execute("INSERT INTO tblTrustListInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" inserted TrustList xml"); + return true; + } + + if(message.GetName()=="PutFailed") + { + m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='false' WHERE LocalIdentityID="+idparts[1]+";"); + m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" failure inserting TrustList xml. Code="+message["Code"]+" Description="+message["CodeDescription"]); + + // if code 9 (collision), then insert index into inserted table + if(message["Code"]=="9") + { + m_db->Execute("INSERT INTO tblTrustListInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + } + + return true; + } + + } + + return false; +} + +void TrustListInserter::Initialize() +{ + Option::instance()->Get("MessageBase",m_messagebase); + m_lastchecked.SetToGMTime(); +} + +void TrustListInserter::Process() +{ + DateTime now; + now.SetToGMTime(); + + // check every minute + if(m_lastchecked<=(now-(1.0/1440.0))) + { + CheckForNeededInsert(); + m_lastchecked=now; + } +} + +void TrustListInserter::RegisterWithThread(FreenetMasterThread *thread) +{ + thread->RegisterFCPConnected(this); + thread->RegisterFCPMessageHandler(this); + thread->RegisterPeriodicProcessor(this); +} + +void TrustListInserter::StartInsert(const long localidentityid, const std::string &privatekey) +{ + FCPMessage message; + TrustListXML xml; + std::string data; + std::string datasizestr; + std::string publickey; + int messagetrust; + int trustlisttrust; + DateTime now; + int index; + std::string indexstr; + std::string localidentityidstr; + + now.SetToGMTime(); + + // build the xml file + SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey, LocalMessageTrust, LocalTrustListTrust FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey<>'';"); + st.Step(); + while(st.RowReturned()) + { + st.ResultText(0,publickey); + st.ResultInt(1,messagetrust); + st.ResultInt(2,trustlisttrust); + xml.AddTrust(publickey,messagetrust,trustlisttrust); + st.Step(); + } + + // get next insert index + st=m_db->Prepare("SELECT MAX(InsertIndex) FROM tblTrustListInserts WHERE LocalIdentityID=? AND Day=?;"); + st.Bind(0,localidentityid); + st.Bind(1,now.Format("%Y-%m-%d")); + st.Step(); + + index=0; + if(st.RowReturned() && st.ResultNull(0)==false) + { + st.ResultInt(0,index); + index++; + } + + StringFunctions::Convert(localidentityid,localidentityidstr); + StringFunctions::Convert(index,indexstr); + + data=xml.GetXML(); + StringFunctions::Convert(data.size(),datasizestr); + + message.SetName("ClientPut"); + message["URI"]=privatekey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; + message["Identifier"]="TrustListInserter|"+localidentityidstr+"|"+indexstr+"|"+message["URI"]; + message["UploadFrom"]="direct"; + message["DataLength"]=datasizestr; + m_fcp->SendMessage(message); + m_fcp->SendRaw(data.c_str(),data.size()); + + m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='true' WHERE LocalIdentityID="+localidentityidstr+";"); + +} \ No newline at end of file diff --git a/src/freenet/trustlistrequester.cpp b/src/freenet/trustlistrequester.cpp new file mode 100644 index 0000000..f5e4be9 --- /dev/null +++ b/src/freenet/trustlistrequester.cpp @@ -0,0 +1,329 @@ +#include "../../include/freenet/trustlistrequester.h" +#include "../../include/option.h" +#include "../../include/stringfunctions.h" +#include "../../include/freenet/trustlistxml.h" + +#ifdef XMEM + #include +#endif + +TrustListRequester::TrustListRequester() +{ + Initialize(); +} + +TrustListRequester::TrustListRequester(FCPv2 *fcp):IFCPConnected(fcp) +{ + Initialize(); +} + +void TrustListRequester::FCPConnected() +{ + m_requesting.clear(); + PopulateIDList(); +} + +void TrustListRequester::FCPDisconnected() +{ + +} + +const bool TrustListRequester::HandleAllData(FCPMessage &message) +{ + DateTime now; + SQLite3DB::Statement st; + std::vector idparts; + long datalength; + std::vector data; + TrustListXML xml; + long identityid; + long index; + + now.SetToGMTime(); + StringFunctions::Split(message["Identifier"],"|",idparts); + StringFunctions::Convert(message["DataLength"],datalength); + StringFunctions::Convert(idparts[1],identityid); + StringFunctions::Convert(idparts[2],index); + + // wait for all data to be received from connection + while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()Update(1); + } + + // if we got disconnected- return immediately + if(m_fcp->Connected()==false) + { + return false; + } + + // receive the file + data.resize(datalength); + m_fcp->ReceiveRaw(&data[0],datalength); + + // parse file into xml and update the database + if(xml.ParseXML(std::string(data.begin(),data.end()))==true) + { + st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;"); + // loop through all trust entries in xml and add to database if we don't already know them + for(long i=0; iExecute("INSERT INTO tblIdentity(PublicKey,DateAdded) VALUES('"+identity+"','"+now.Format("%Y-%m-%d %H:%M:%S")+"');"); + } + st.Reset(); + } + + st=m_db->Prepare("INSERT INTO tblTrustListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');"); + st.Bind(0,identityid); + st.Bind(1,idparts[4]); + st.Bind(2,index); + st.Step(); + st.Finalize(); + + m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,__FUNCTION__" parsed Identity XML file : "+message["Identifier"]); + } + else + { + // bad data - mark index + st=m_db->Prepare("INSERT INTO tblTrustListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');"); + st.Bind(0,identityid); + st.Bind(1,idparts[4]); + st.Bind(2,index); + st.Step(); + st.Finalize(); + + m_log->WriteLog(LogFile::LOGLEVEL_ERROR,__FUNCTION__" error parsing TrustList XML file : "+message["Identifier"]); + } + + // remove this identityid from request list + RemoveFromRequestList(identityid); + + return true; + +} + +const bool TrustListRequester::HandleGetFailed(FCPMessage &message) +{ + DateTime now; + SQLite3DB::Statement st; + std::vector idparts; + long identityid; + long index; + + now.SetToGMTime(); + StringFunctions::Split(message["Identifier"],"|",idparts); + StringFunctions::Convert(idparts[1],identityid); + StringFunctions::Convert(idparts[2],index); + + // if this is a fatal error - insert index into database so we won't try to download this index again + if(message["Fatal"]=="true") + { + st=m_db->Prepare("INSERT INTO tblTrustListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'false');"); + st.Bind(0,identityid); + st.Bind(1,idparts[4]); + st.Bind(2,index); + st.Step(); + st.Finalize(); + + m_log->WriteLog(LogFile::LOGLEVEL_ERROR,__FUNCTION__" fatal error requesting "+message["Identifier"]); + } + + // remove this identityid from request list + RemoveFromRequestList(identityid); + + return true; + +} + +const bool TrustListRequester::HandleMessage(FCPMessage &message) +{ + + if(message["Identifier"].find("TrustListRequester")==0) + { + if(message.GetName()=="DataFound") + { + return true; + } + + if(message.GetName()=="AllData") + { + return HandleAllData(message); + } + + if(message.GetName()=="GetFailed") + { + return HandleGetFailed(message); + } + + if(message.GetName()=="IdentifierCollision") + { + // remove one of the ids from the requesting list + long identityid=0; + std::vector idparts; + StringFunctions::Split(message["Identifier"],"|",idparts); + StringFunctions::Convert(idparts[1],identityid); + RemoveFromRequestList(identityid); + return true; + } + } + + return false; +} + +void TrustListRequester::Initialize() +{ + std::string tempval=""; + Option::instance()->Get("MaxIdentityRequests",tempval); + StringFunctions::Convert(tempval,m_maxrequests); + if(m_maxrequests<1) + { + m_maxrequests=1; + m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"Option MaxTrustListRequests is currently set at "+tempval+". It must be 1 or greater."); + } + if(m_maxrequests>100) + { + m_log->WriteLog(LogFile::LOGLEVEL_WARNING,"Option MaxTrustListRequests is currently set at "+tempval+". This value might be incorrectly configured."); + } + Option::instance()->Get("MessageBase",m_messagebase); + m_tempdate.SetToGMTime(); +} + +void TrustListRequester::PopulateIDList() +{ + DateTime date; + int id; + + date.SetToGMTime(); + + // select identities we want to query (we've seen them today and they are publishing trust list) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) + SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+date.Format("%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st.Step(); + + m_ids.clear(); + + while(st.RowReturned()) + { + st.ResultInt(0,id); + m_ids[id]=false; + st.Step(); + } +} + +void TrustListRequester::Process() +{ + // max is the smaller of the config value or the total number of identities we will request from + long max=m_maxrequests>m_ids.size() ? m_ids.size() : m_maxrequests; + + // try to keep up to max requests going + if(m_requesting.size()::iterator i=m_ids.begin(); + while(i!=m_ids.end() && (*i).second==true) + { + i++; + } + + if(i!=m_ids.end()) + { + StartRequest((*i).first); + } + else + { + // we requested from all ids in the list, repopulate the list + PopulateIDList(); + } + } + // special case - if there were 0 identities on the list when we started then we will never get a chance to repopulate the list + // this will recheck for ids every minute + DateTime now; + now.SetToGMTime(); + if(m_tempdate<(now-(1.0/1440.0))) + { + PopulateIDList(); + m_tempdate=now; + } + +} + +void TrustListRequester::RegisterWithThread(FreenetMasterThread *thread) +{ + thread->RegisterFCPConnected(this); + thread->RegisterFCPMessageHandler(this); + thread->RegisterPeriodicProcessor(this); +} + +void TrustListRequester::RemoveFromRequestList(const long identityid) +{ + std::vector::iterator i=m_requesting.begin(); + while(i!=m_requesting.end() && (*i)!=identityid) + { + i++; + } + if(i!=m_requesting.end()) + { + m_requesting.erase(i); + } +} + +void TrustListRequester::StartRequest(const long identityid) +{ + DateTime now; + FCPMessage message; + std::string publickey; + int index; + std::string indexstr; + std::string identityidstr; + + SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;"); + st.Bind(0,identityid); + st.Step(); + + if(st.RowReturned()) + { + st.ResultText(0,publickey); + + now.SetToGMTime(); + + SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblTrustListRequests WHERE Day=? AND IdentityID=?;"); + st2.Bind(0,now.Format("%Y-%m-%d")); + st2.Bind(1,identityid); + st2.Step(); + + index=0; + if(st2.RowReturned()) + { + if(st2.ResultNull(0)==false) + { + st2.ResultInt(0,index); + index++; + } + } + st2.Finalize(); + + StringFunctions::Convert(index,indexstr); + StringFunctions::Convert(identityid,identityidstr); + + message.SetName("ClientGet"); + message["URI"]=publickey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|TrustList|"+indexstr+".xml"; + message["Identifier"]="TrustListRequester|"+identityidstr+"|"+indexstr+"|"+message["URI"]; + message["ReturnType"]="direct"; + message["MaxSize"]="1000000"; // 1 MB + + m_fcp->SendMessage(message); + + m_requesting.push_back(identityid); + } + st.Finalize(); + + m_ids[identityid]=true; + +} diff --git a/src/freenet/trustlistxml.cpp b/src/freenet/trustlistxml.cpp new file mode 100644 index 0000000..45af43e --- /dev/null +++ b/src/freenet/trustlistxml.cpp @@ -0,0 +1,148 @@ +#include "../../include/freenet/trustlistxml.h" +#include "../../include/stringfunctions.h" + +#ifdef XMEM + #include +#endif + +TrustListXML::TrustListXML() +{ + Initialize(); +} + +void TrustListXML::AddTrust(const std::string &identity, const long messagetrust, const long trustlisttrust) +{ + m_trust.push_back(trust(identity,messagetrust,trustlisttrust)); +} + +std::string TrustListXML::GetIdentity(const long index) +{ + if(index>=0 && index=0 && index=0 && index::iterator i=m_trust.begin(); i!=m_trust.end(); i++) + { + std::string messagetrust; + std::string trustlisttrust; + StringFunctions::Convert((*i).m_messagetrust,messagetrust); + StringFunctions::Convert((*i).m_trustlisttrust,trustlisttrust); + TiXmlElement *tr=new TiXmlElement("Trust"); + tid->LinkEndChild(tr); + tr->LinkEndChild(XMLCreateCDATAElement("Identity",(*i).m_identity)); + tr->LinkEndChild(XMLCreateTextElement("MessageTrustLevel",messagetrust)); + tr->LinkEndChild(XMLCreateTextElement("TrustListTrustLevel",trustlisttrust)); + } + + td.Accept(&tp); + return std::string(tp.CStr()); +} + +void TrustListXML::Initialize() +{ + m_trust.clear(); +} + +const bool TrustListXML::ParseXML(const std::string &xml) +{ + TiXmlDocument td; + td.Parse(xml.c_str()); + + if(!td.Error()) + { + std::string identity; + std::string messagetruststr; + std::string trustlisttruststr; + long messagetrust; + long trustlisttrust; + TiXmlText *txt; + TiXmlHandle hnd(&td); + TiXmlNode *node; + + Initialize(); + + node=hnd.FirstChild("TrustList").FirstChild("Trust").ToElement(); + while(node) + { + identity=""; + messagetrust=-1; + trustlisttrust=-1; + + TiXmlHandle hnd2(node); + txt=hnd2.FirstChild("Identity").FirstChild().ToText(); + if(txt) + { + identity=txt->ValueStr(); + } + txt=hnd2.FirstChild("MessageTrustLevel").FirstChild().ToText(); + if(txt) + { + messagetruststr=txt->ValueStr(); + StringFunctions::Convert(messagetruststr,messagetrust); + } + txt=hnd2.FirstChild("TrustListTrustLevel").FirstChild().ToText(); + if(txt) + { + trustlisttruststr=txt->ValueStr(); + StringFunctions::Convert(trustlisttruststr,trustlisttrust); + } + + if(identity!="" && messagetrust>=0 && messagetrust<=100 && trustlisttrust>=0 && trustlisttrust<=100) + { + m_trust.push_back(trust(identity,messagetrust,trustlisttrust)); + } + else + { + m_log->WriteLog(LogFile::LOGLEVEL_ERROR,__FUNCTION__," malformed Trust in TrustList.xml"); + } + + node=node->NextSibling("Trust"); + } + + } + else + { + return false; + } +} diff --git a/src/identitytestglobal.cpp b/src/identitytestglobal.cpp index 2acca4d..9297101 100644 --- a/src/identitytestglobal.cpp +++ b/src/identitytestglobal.cpp @@ -35,12 +35,16 @@ void SetupDB() PublicKey TEXT,\ PrivateKey TEXT,\ SingleUse BOOL CHECK(SingleUse IN('true','false')) DEFAULT 'false',\ + PublishTrustList BOOL CHECK(PublishTrustList IN('true','false')) DEFAULT 'false',\ + PublishBoardList BOOL CHECK(PublishBoardList IN('true','false')) DEFAULT 'false',\ InsertingIdentity BOOL CHECK(InsertingIdentity IN('true','false')) DEFAULT 'false',\ LastInsertedIdentity DATETIME,\ InsertingPuzzle BOOL CHECK(InsertingPuzzle IN('true','false')) DEFAULT 'false',\ LastInsertedPuzzle DATETIME,\ InsertingTrustList BOOL CHECK(InsertingTrustList IN('true','false')) DEFAULT 'false',\ - LastInsertedTrustList DATETIME\ + LastInsertedTrustList DATETIME,\ + InsertingBoardList BOOL CHECK(InsertingBoardList IN('true','false')) DEFAULT 'false',\ + LastInsertedBoardList DATETIME\ );"); db->Execute("CREATE TABLE IF NOT EXISTS tblLocalIdentityInserts(\ @@ -49,6 +53,19 @@ void SetupDB() InsertIndex INTEGER\ );"); + db->Execute("CREATE TABLE IF NOT EXISTS tblTrustListInserts(\ + LocalIdentityID INTEGER,\ + Day DATE,\ + InsertIndex INTEGER\ + );"); + + db->Execute("CREATE TABLE IF NOT EXISTS tblTrustListRequests(\ + IdentityID INTEGER,\ + Day DATE,\ + RequestIndex INTEGER,\ + Found BOOL CHECK(Found IN('true','false')) DEFAULT 'false'\ + );"); + db->Execute("CREATE TABLE IF NOT EXISTS tblIntroductionPuzzleInserts(\ UUID TEXT UNIQUE,\ LocalIdentityID INTEGER,\ @@ -66,6 +83,8 @@ void SetupDB() PublicKey TEXT,\ Name TEXT,\ SingleUse BOOL CHECK(SingleUse IN('true','false')) DEFAULT 'false',\ + PublishTrustList BOOL CHECK(PublishTrustList IN('true','false')) DEFAULT 'false',\ + PublishBoardList BOOL CHECK(PublishBoardList IN('true','false')) DEFAULT 'false',\ DateAdded DATETIME,\ LastSeen DATETIME,\ LocalMessageTrust INTEGER CHECK(LocalMessageTrust BETWEEN 0 AND 100) DEFAULT 50,\ @@ -75,7 +94,7 @@ void SetupDB() );"); db->Execute("CREATE TABLE IF NOT EXISTS tblIdentityRequests(\ - IdentityID INTEGER PRIMARY KEY,\ + IdentityID INTEGER,\ Day DATE,\ RequestIndex INTEGER,\ Found BOOL CHECK(Found IN('true','false')) DEFAULT 'false'\ @@ -100,6 +119,51 @@ void SetupDB() Inserted BOOL CHECK(Inserted IN('true','false')) DEFAULT 'false'\ );"); + db->Execute("CREATE TABLE IF NOT EXISTS tblPeerTrust(\ + IdentityID INTEGER,\ + TargetIdentityID INTEGER,\ + MessageTrust INTEGER CHECK(MessageTrust BETWEEN 0 AND 100),\ + TrustListTrust INTEGER CHECK(TrustListTrust BETWEEN 0 AND 100)\ + );"); + + // calculates peer trust + db->Execute("CREATE VIEW IF NOT EXISTS vwCalculatedPeerTrust AS \ + SELECT TargetIdentityID, \ + ROUND(SUM(MessageTrust*(LocalMessageTrust/100.0))/SUM(LocalMessageTrust/100.0),0) AS 'PeerMessageTrust', \ + ROUND(SUM(TrustListTrust*(LocalTrustListTrust/100.0))/SUM(LocalTrustListTrust/100.0),0) AS 'PeerTrustListTrust' \ + FROM tblPeerTrust INNER JOIN tblIdentity ON tblPeerTrust.IdentityID=tblIdentity.IdentityID \ + WHERE LocalTrustListTrust>(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') \ + GROUP BY TargetIdentityID;"); + + // update PeerTrustLevel when deleting a record from tblPeerTrust + db->Execute("CREATE TRIGGER trgDeleteOntblPeerTrust AFTER DELETE ON tblPeerTrust \ + FOR EACH ROW \ + BEGIN \ + UPDATE tblIdentity SET PeerMessageTrust=(SELECT PeerMessageTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=old.TargetIdentityID), PeerTrustListTrust=(SELECT PeerTrustListTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=old.TargetIdentityID) WHERE IdentityID=old.TargetIdentityID;\ + END;"); + + // update PeerTrustLevel when inserting a record into tblPeerTrust + db->Execute("CREATE TRIGGER trgInsertOntblPeerTrust AFTER INSERT ON tblPeerTrust \ + FOR EACH ROW \ + BEGIN \ + UPDATE tblIdentity SET PeerMessageTrust=(SELECT PeerMessageTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=new.TargetIdentityID), PeerTrustListTrust=(SELECT PeerTrustListTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=new.TargetIdentityID) WHERE IdentityID=new.TargetIdentityID;\ + END;"); + + // update PeerTrustLevel when updating a record in tblPeerTrust + db->Execute("CREATE TRIGGER trgUpdateOntblPeerTrust AFTER UPDATE ON tblPeerTrust \ + FOR EACH ROW \ + BEGIN \ + UPDATE tblIdentity SET PeerMessageTrust=(SELECT PeerMessageTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=old.TargetIdentityID), PeerTrustListTrust=(SELECT PeerTrustListTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=old.TargetIdentityID) WHERE IdentityID=old.TargetIdentityID;\ + UPDATE tblIdentity SET PeerMessageTrust=(SELECT PeerMessageTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=new.TargetIdentityID), PeerTrustListTrust=(SELECT PeerTrustListTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=new.TargetIdentityID) WHERE IdentityID=new.TargetIdentityID;\ + END;"); + + // recalculate all Peer TrustLevels when updating Local TrustLevels on tblIdentity - doesn't really need to be all, but rather all identities the updated identity has a trust level for. It's easier to update everyone for now. + db->Execute("CREATE TRIGGER trgUpdateLocalTrustLevels AFTER UPDATE OF LocalMessageTrust,LocalTrustListTrust ON tblIdentity \ + FOR EACH ROW \ + BEGIN \ + UPDATE tblIdentity SET PeerMessageTrust=(SELECT PeerMessageTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=IdentityID), PeerTrustListTrust=(SELECT PeerTrustListTrust FROM vwCalculatedPeerTrust WHERE TargetIdentityID=IdentityID);\ + END;"); + } void SetupDefaultOptions() @@ -164,6 +228,18 @@ void SetupDefaultOptions() st.Step(); st.Reset(); + st.Bind(0,"MaxTrustListRequests"); + st.Bind(1,"5"); + st.Bind(2,"Maximum number of concurrent requests for new Trust Lists"); + st.Step(); + st.Reset(); + + st.Bind(0,"MinLocalTrustListTrust"); + st.Bind(1,"50"); + st.Bind(2,"Specifies a local trust list trust level that a peer must have before its trust list will be included in the weighted average. Any peers below this number will be excluded from the results."); + st.Step(); + st.Reset(); + } void SetupLogFile() diff --git a/src/identitytestmain.cpp b/src/identitytestmain.cpp index 4776e36..21244df 100644 --- a/src/identitytestmain.cpp +++ b/src/identitytestmain.cpp @@ -21,7 +21,6 @@ int main() SetupDB(); SetupDefaultOptions(); - SetupLogFile(); SetupNetwork(); -- 2.7.4