From: SomeDude Date: Sat, 7 Feb 2009 10:23:00 +0000 (+0100) Subject: version 0.3.31 X-Git-Url: https://git.pterodactylus.net/?p=fms.git;a=commitdiff_plain;h=ed0732b2550c23c05fc9faf925620e87ee6dee12 version 0.3.31 --- diff --git a/include/dbconversions.h b/include/dbconversions.h index 597e171..0406e7f 100644 --- a/include/dbconversions.h +++ b/include/dbconversions.h @@ -17,6 +17,7 @@ void ConvertDB0111To0112(SQLite3DB::DB *db); void ConvertDB0112To0113(SQLite3DB::DB *db); void ConvertDB0113To0114(SQLite3DB::DB *db); void ConvertDB0114To0115(SQLite3DB::DB *db); +void ConvertDB0115To0116(SQLite3DB::DB *db); // TODO remove sometime after 0.1.17 void FixCapitalBoardNames(SQLite3DB::DB *db); diff --git a/include/freenet/iindexinserter.h b/include/freenet/iindexinserter.h index 2135e45..bbbd31a 100644 --- a/include/freenet/iindexinserter.h +++ b/include/freenet/iindexinserter.h @@ -136,12 +136,10 @@ void IIndexInserter::InitializeIIndexInserter() template void IIndexInserter::Process() { - Poco::DateTime now; - - if(m_lastchecked<(now-Poco::Timespan(0,0,1,0,0))) + if(m_lastchecked<(Poco::DateTime()-Poco::Timespan(0,0,1,0,0))) { CheckForNeededInsert(); - m_lastchecked=now; + m_lastchecked=Poco::DateTime(); } } diff --git a/include/freenet/messagelistinserter.h b/include/freenet/messagelistinserter.h index c13383a..0b0c0ac 100644 --- a/include/freenet/messagelistinserter.h +++ b/include/freenet/messagelistinserter.h @@ -18,6 +18,7 @@ private: long m_daysbackward; std::map m_lastinsertedxml; // last xml document inserted for each local identity + Poco::DateTime m_laststartedinsert; }; diff --git a/include/freenet/messagelistrequester.h b/include/freenet/messagelistrequester.h index 079fbe7..fdeaf4d 100644 --- a/include/freenet/messagelistrequester.h +++ b/include/freenet/messagelistrequester.h @@ -4,6 +4,9 @@ #include "iindexrequester.h" #include +#include + +#include class MessageListRequester:public IIndexRequester { @@ -18,7 +21,7 @@ private: void StartRedirectRequest(FCPv2::Message &message); const bool HandleAllData(FCPv2::Message &message); const bool HandleGetFailed(FCPv2::Message &message); - void GetBoardList(std::map &boards); + void GetBoardList(std::map &boards, const bool forceload=false); const bool CheckDateNotFuture(const std::string &datestr) const; const bool CheckDateWithinMaxDays(const std::string &datestr) const; @@ -26,6 +29,11 @@ private: bool m_savetonewboards; long m_messagedownloadmaxdaysbackward; + std::map m_boardscache; + Poco::DateTime m_boardscacheupdate; // last time we updated the boards cache + + std::map > > m_requestindexcache; // date - identity id - index + }; #endif // _messagelistrequester_ diff --git a/include/global.h b/include/global.h index 4fc4ad5..415d3c6 100644 --- a/include/global.h +++ b/include/global.h @@ -7,10 +7,10 @@ #define VERSION_MAJOR "0" #define VERSION_MINOR "3" -#define VERSION_RELEASE "29" +#define VERSION_RELEASE "31" #define FMS_VERSION VERSION_MAJOR"."VERSION_MINOR"."VERSION_RELEASE -#define FMS_FREESITE_USK "USK@0npnMrqZNKRCRoGojZV93UNHCMN-6UU3rRSAmP6jNLE,~BG-edFtdCC1cSH4O3BWdeIYa8Sw5DfyrSV-TKdO5ec,AQACAAE/fms/91/" -#define FMS_VERSION_EDITION "31" +#define FMS_FREESITE_USK "USK@0npnMrqZNKRCRoGojZV93UNHCMN-6UU3rRSAmP6jNLE,~BG-edFtdCC1cSH4O3BWdeIYa8Sw5DfyrSV-TKdO5ec,AQACAAE/fms/93/" +#define FMS_VERSION_EDITION "33" typedef Poco::ScopedLock Guard; diff --git a/include/ifmsxmldocument.h b/include/ifmsxmldocument.h index ae3f4e6..0a0ddc1 100644 --- a/include/ifmsxmldocument.h +++ b/include/ifmsxmldocument.h @@ -209,13 +209,20 @@ protected: return returntext; } - const std::string GenerateXML(Poco::AutoPtr doc) + const std::string GenerateXML(Poco::AutoPtr doc, const bool prettyprint=true) { std::ostringstream str; if(doc) { Poco::XML::DOMWriter dr; - dr.setOptions(Poco::XML::XMLWriter::WRITE_XML_DECLARATION | Poco::XML::XMLWriter::PRETTY_PRINT); + if(prettyprint==true) + { + dr.setOptions(Poco::XML::XMLWriter::WRITE_XML_DECLARATION | Poco::XML::XMLWriter::PRETTY_PRINT); + } + else + { + dr.setOptions(Poco::XML::XMLWriter::WRITE_XML_DECLARATION); + } dr.setNewLine(Poco::XML::XMLWriter::NEWLINE_CRLF); dr.writeNode(str,doc); } diff --git a/src/dbconversions.cpp b/src/dbconversions.cpp index 78d906a..463cf7e 100644 --- a/src/dbconversions.cpp +++ b/src/dbconversions.cpp @@ -241,6 +241,17 @@ void ConvertDB0114To0115(SQLite3DB::DB *db) db->Execute("UPDATE tblDBVersion SET Major=1, Minor=15;"); } +void ConvertDB0115To0116(SQLite3DB::DB *db) +{ + // Add FromIdentityID to tblMessageRequests so we know who we got this index from + // Add FailureCount + + db->Execute("ALTER TABLE tblMessageRequests ADD COLUMN FromIdentityID INTEGER;"); + db->Execute("ALTER TABLE tblIdentity ADD COLUMN FailureCount INTEGER CHECK(FailureCount>=0) DEFAULT 0;"); + + db->Execute("UPDATE tblDBVersion SET Major=1, Minor=16;"); +} + void FixCapitalBoardNames(SQLite3DB::DB *db) { diff --git a/src/dbmaintenancethread.cpp b/src/dbmaintenancethread.cpp index e9f03e0..0b1349f 100644 --- a/src/dbmaintenancethread.cpp +++ b/src/dbmaintenancethread.cpp @@ -79,6 +79,12 @@ void DBMaintenanceThread::Do10MinuteMaintenance() { std::string dbres=TestDBIntegrity(m_db); m_log->trace("DBMaintenanceThread::Do10MinuteMaintenance() middle TestDBIntegrity returned "+dbres); + if(dbres!="ok") + { + m_db->Execute("REINDEX;"); + dbres=TestDBIntegrity(m_db); + m_log->trace("DBMaintenanceThread::Do10MinuteMaintenenace() middle after reindex returned "+dbres); + } } // now rebuild threads where the message has been deleted @@ -174,6 +180,9 @@ void DBMaintenanceThread::Do1HourMaintenance() st.Step(); } + st.Finalize(); + upd.Finalize(); + // insert all identities not in trust list already m_db->Execute("INSERT INTO tblIdentityTrust(LocalIdentityID,IdentityID) SELECT LocalIdentityID,IdentityID FROM tblLocalIdentity,tblIdentity WHERE LocalIdentityID || '_' || IdentityID NOT IN (SELECT LocalIdentityID || '_' || IdentityID FROM tblIdentityTrust);"); @@ -215,6 +224,10 @@ void DBMaintenanceThread::Do6HourMaintenance() st.Step(); } + st.Finalize(); + st2.Finalize(); + upd.Finalize(); + m_db->Execute("COMMIT;"); m_log->debug("PeriodicDBMaintenance::Do6HourMaintenance"); @@ -274,6 +287,7 @@ void DBMaintenanceThread::Do1DayMaintenance() // try to re-attach messages from identities that were previously deleted, but have been since re-added // first get the names from messages that have a NULL IdentityID SQLite3DB::Statement st=m_db->Prepare("SELECT FromName FROM tblMessage WHERE IdentityID IS NULL GROUP BY FromName;"); + SQLite3DB::Statement findst=m_db->Prepare("SELECT IdentityID,PublicKey FROM tblIdentity WHERE Name=?;"); st.Step(); while(st.RowReturned()) { @@ -297,27 +311,27 @@ void DBMaintenanceThread::Do1DayMaintenance() } // find identities with this name - SQLite3DB::Statement st2=m_db->Prepare("SELECT IdentityID,PublicKey FROM tblIdentity WHERE Name=?;"); - st2.Bind(0,namepart); - st2.Step(); - while(st2.RowReturned()) + findst.Bind(0,namepart); + findst.Step(); + while(findst.RowReturned()) { publickey=""; identityid=0; - st2.ResultText(1,publickey); + findst.ResultText(1,publickey); // check if public key matches 2nd part if(parts.size()>1 && publickey.find(parts[1])==4) { // we have the identity - so update the messages table with the identityid - st2.ResultInt(0,identityid); + findst.ResultInt(0,identityid); SQLite3DB::Statement st3=m_db->Prepare("UPDATE tblMessage SET IdentityID=? WHERE FromName=? AND IdentityID IS NULL;"); st3.Bind(0,identityid); st3.Bind(1,name); st3.Step(); } - st2.Step(); + findst.Step(); } + findst.Reset(); st.Step(); } @@ -355,6 +369,14 @@ void DBMaintenanceThread::Do1DayMaintenance() m_db->Execute("DELETE FROM tblIdentityTrust WHERE LocalIdentityID NOT IN (SELECT LocalIdentityID FROM tblLocalIdentity);"); m_db->Execute("DELETE FROM tblIdentityTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);"); + + // reduce failure count for each identity + m_db->Execute("UPDATE tblIdentity SET FailureCount=0 WHERE FailureCount<(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');"); + m_db->Execute("UPDATE tblIdentity SET FailureCount=FailureCount-(SELECT OptionValue FROM tblOption WHERE OptionName='FailureCountReduction') WHERE FailureCount>=(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');"); + + st.Finalize(); + findst.Finalize(); + m_db->Execute("COMMIT;"); m_log->debug("PeriodicDBMaintenance::Do1DayMaintenance"); diff --git a/src/dbsetup.cpp b/src/dbsetup.cpp index e763469..644310b 100644 --- a/src/dbsetup.cpp +++ b/src/dbsetup.cpp @@ -111,13 +111,19 @@ void SetupDB(SQLite3DB::DB *db) major=1; minor=15; } + if(major==1 && minor==15) + { + ConvertDB0115To0116(db); + major=1; + minor=16; + } } else { - db->Execute("INSERT INTO tblDBVersion(Major,Minor) VALUES(1,15);"); + db->Execute("INSERT INTO tblDBVersion(Major,Minor) VALUES(1,16);"); } - db->Execute("UPDATE tblDBVersion SET Major=1, Minor=15;"); + db->Execute("UPDATE tblDBVersion SET Major=1, Minor=16;"); db->Execute("CREATE TABLE IF NOT EXISTS tblFMSVersion(\ Major INTEGER,\ @@ -218,7 +224,8 @@ void SetupDB(SQLite3DB::DB *db) PeerTrustListTrust INTEGER CHECK(PeerTrustListTrust BETWEEN 0 AND 100) DEFAULT NULL,\ AddedMethod TEXT,\ Hidden BOOL CHECK(Hidden IN('true','false')) DEFAULT 'false',\ - PurgeDate DATETIME\ + PurgeDate DATETIME,\ + FailureCount INTEGER CHECK(FailureCount>=0) DEFAULT 0\ );"); db->Execute("CREATE TABLE IF NOT EXISTS tblIdentityRequests(\ @@ -353,7 +360,8 @@ void SetupDB(SQLite3DB::DB *db) FromMessageList BOOL CHECK(FromMessageList IN('true','false')) DEFAULT 'false',\ Found BOOL CHECK(Found IN('true','false')) DEFAULT 'false',\ Tries INTEGER DEFAULT 0,\ - Key TEXT\ + Key TEXT,\ + FromIdentityID INTEGER\ );"); db->Execute("CREATE UNIQUE INDEX IF NOT EXISTS idxMessageRequest ON tblMessageRequests(IdentityID,Day,RequestIndex);"); diff --git a/src/freenet/boardlistrequester.cpp b/src/freenet/boardlistrequester.cpp index 41df7d6..802786c 100644 --- a/src/freenet/boardlistrequester.cpp +++ b/src/freenet/boardlistrequester.cpp @@ -236,11 +236,11 @@ void BoardListRequester::PopulateIDList() if(m_localtrustoverrides==false) { - st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND PublishBoardList='true' ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND PublishBoardList='true' AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); } else { - st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND PublishBoardList='true' ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND PublishBoardList='true' AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); } st.Step(); @@ -264,7 +264,7 @@ void BoardListRequester::StartRequest(const long &identityid) int index; std::string identityidstr; - SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE identityid=?;"); + SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;"); st.Bind(0,identityid); st.Step(); diff --git a/src/freenet/identityinserter.cpp b/src/freenet/identityinserter.cpp index 1ae04a5..85fe2f4 100644 --- a/src/freenet/identityinserter.cpp +++ b/src/freenet/identityinserter.cpp @@ -84,29 +84,45 @@ const bool IdentityInserter::HandleMessage(FCPv2::Message &message) Poco::DateTime lastdate; int tzdiff=0; Poco::DateTimeParser::tryParse("%Y-%m-%d",idparts[4],lastdate,tzdiff); - - if(lastdate.day()==now.day()) + + // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now + if(message["Identifier"].find(".xml")!=std::string::npos) { - m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false', LastInsertedIdentity='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";"); + if(lastdate.day()==now.day()) + { + m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false', LastInsertedIdentity='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";"); + } + else + { + m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";"); + } + m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + m_log->debug("IdentityInserter::HandleMessage inserted Identity xml"); } else { - m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";"); + m_log->trace("IdentityInserter::HandleMessage inserted editioned Identity xml"); } - m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); - m_log->debug("IdentityInserter::HandleMessage inserted Identity xml"); return true; } if(message.GetName()=="PutFailed") { - m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";"); - m_log->debug("IdentityInserter::HandleMessage failure inserting Identity xml. Code="+message["Code"]+" Description="+message["CodeDescription"]); - - // if code 9 (collision), then insert index into inserted table - if(message["Code"]=="9") + // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now + if(message["Identifier"].find(".xml")!=std::string::npos) { - m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";"); + m_log->debug("IdentityInserter::HandleMessage failure inserting Identity xml. Code="+message["Code"]+" Description="+message["CodeDescription"]); + + // if code 9 (collision), then insert index into inserted table + if(message["Code"]=="9") + { + m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");"); + } + } + else + { + m_log->trace("IdentityInserter::HandleMessage PutFailed for editioned SSK error code "+message["Code"]+ " id "+message["Identifier"]); } return true; @@ -236,6 +252,16 @@ void IdentityInserter::StartInsert(const long localidentityid) m_fcp->Send(mess); m_fcp->Send(std::vector(data.begin(),data.end())); + // test insert as editioned SSK + mess.Clear(); + mess.SetName("ClientPut"); + mess["URI"]=privatekey+messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|Identity|-"+indexstr; + mess["Identifier"]="IdentityInserter|"+mess["URI"]; + mess["UploadFrom"]="direct"; + mess["DataLength"]=datasizestr; + m_fcp->Send(mess); + m_fcp->Send(std::vector(data.begin(),data.end())); + m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='true' WHERE LocalIdentityID="+idstring+";"); } diff --git a/src/freenet/identityrequester.cpp b/src/freenet/identityrequester.cpp index 18810d5..6744c5a 100644 --- a/src/freenet/identityrequester.cpp +++ b/src/freenet/identityrequester.cpp @@ -188,7 +188,7 @@ void IdentityRequester::PopulateIDList() date.assign(date.year(),date.month(),date.day(),0,0,0); // select identities we want to query (haven't seen yet today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) - SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen IS NOT NULL AND LastSeen<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S")+"' ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen IS NOT NULL AND LastSeen<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S")+"' AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); st.Step(); m_ids.clear(); diff --git a/src/freenet/introductionpuzzleinserter.cpp b/src/freenet/introductionpuzzleinserter.cpp index 000d973..0070ec6 100644 --- a/src/freenet/introductionpuzzleinserter.cpp +++ b/src/freenet/introductionpuzzleinserter.cpp @@ -261,7 +261,8 @@ const bool IntroductionPuzzleInserter::StartInsert(const long &localidentityid) m_fcp->Send(message); m_fcp->Send(std::vector(xmldata.begin(),xmldata.end())); - // insert to USK + // insert to USK - not used, but don't remove code yet + /* message.Clear(); message.SetName("ClientPutComplexDir"); message["URI"]="USK"+privatekey.substr(3)+messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|IntroductionPuzzle/0/"; @@ -272,6 +273,7 @@ const bool IntroductionPuzzleInserter::StartInsert(const long &localidentityid) message["Files.0.DataLength"]=xmldatasizestr; m_fcp->Send(message); m_fcp->Send(std::vector(xmldata.begin(),xmldata.end())); + */ m_db->Execute("INSERT INTO tblIntroductionPuzzleInserts(UUID,Type,MimeType,LocalIdentityID,PuzzleData,PuzzleSolution) VALUES('"+xml.GetUUID()+"','captcha','image/bmp',"+idstring+",'"+encodedpuzzle+"','"+solutionstring+"');"); diff --git a/src/freenet/introductionpuzzlerequester.cpp b/src/freenet/introductionpuzzlerequester.cpp index 9f1ee76..7147e2a 100644 --- a/src/freenet/introductionpuzzlerequester.cpp +++ b/src/freenet/introductionpuzzlerequester.cpp @@ -215,7 +215,7 @@ void IntroductionPuzzleRequester::PopulateIDList() st.Finalize(); // select identities that aren't single use, are publishing a trust list, and have been seen today ( order by trust DESC and limit to limitnum ) - st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublishTrustList='true' AND PublicKey IS NOT NULL AND PublicKey <> '' AND SingleUse='false' AND (LocalTrustListTrust IS NULL OR LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust')) AND LastSeen>='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"' ORDER BY LocalMessageTrust DESC LIMIT 0,"+limitnum+";"); + st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublishTrustList='true' AND PublicKey IS NOT NULL AND PublicKey <> '' AND SingleUse='false' AND (LocalTrustListTrust IS NULL OR LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust')) AND LastSeen>='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"' AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust DESC LIMIT 0,"+limitnum+";"); st.Step(); m_ids.clear(); diff --git a/src/freenet/messageinserter.cpp b/src/freenet/messageinserter.cpp index f3aab00..a378ee1 100644 --- a/src/freenet/messageinserter.cpp +++ b/src/freenet/messageinserter.cpp @@ -51,23 +51,34 @@ const bool MessageInserter::HandlePutFailed(FCPv2::Message &message) int index; int localidentityid; std::vector idparts; - StringFunctions::Split(message["Identifier"],"|",idparts); - StringFunctions::Convert(idparts[2],localidentityid); - StringFunctions::Convert(idparts[3],index); - // fatal put - or data exists - insert bogus index into database so we'll try to insert this message again - if(message["Fatal"]=="true" || message["Code"]=="9") + // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now + if(message["Identifier"].find(".xml")!=std::string::npos) { - SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'true');"); - st.Bind(0,localidentityid); - st.Bind(1,idparts[6]); - st.Bind(2,index); - st.Step(); - } - m_log->trace("MessageInserter::HandlePutFailed error code "+message["Code"]+" fatal="+message["Fatal"]); + StringFunctions::Split(message["Identifier"],"|",idparts); + StringFunctions::Convert(idparts[2],localidentityid); + StringFunctions::Convert(idparts[3],index); - RemoveFromInsertList(idparts[1]); + // fatal put - or data exists - insert bogus index into database so we'll try to insert this message again + if(message["Fatal"]=="true" || message["Code"]=="9") + { + SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'true');"); + st.Bind(0,localidentityid); + st.Bind(1,idparts[6]); + st.Bind(2,index); + st.Step(); + } + + m_log->trace("MessageInserter::HandlePutFailed error code "+message["Code"]+" fatal="+message["Fatal"]); + + RemoveFromInsertList(idparts[1]); + + } + else + { + m_log->trace("MessageInserter::HandlePutFailed for editioned SSK error code "+message["Code"]+ " id "+message["Identifier"]); + } return true; } @@ -80,50 +91,60 @@ const bool MessageInserter::HandlePutSuccessful(FCPv2::Message &message) int index; std::vector idparts; - StringFunctions::Split(message["Identifier"],"|",idparts); - StringFunctions::Convert(idparts[3],index); - StringFunctions::Convert(idparts[2],localidentityid); + // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now + if(message["Identifier"].find(".xml")!=std::string::npos) + { - SQLite3DB::Statement st=m_db->Prepare("UPDATE tblMessageInserts SET Day=?, InsertIndex=?, Inserted='true' WHERE MessageUUID=?;"); - st.Bind(0,idparts[6]); - st.Bind(1,index); - st.Bind(2,idparts[1]); - st.Step(); + StringFunctions::Split(message["Identifier"],"|",idparts); + StringFunctions::Convert(idparts[3],index); + StringFunctions::Convert(idparts[2],localidentityid); - // insert record into temp table so MessageList will be inserted ASAP - date=Poco::Timestamp(); - st=m_db->Prepare("INSERT INTO tmpMessageListInsert(LocalIdentityID,Date) VALUES(?,?);"); - st.Bind(0,localidentityid); - st.Bind(1,Poco::DateTimeFormatter::format(date,"%Y-%m-%d")); - st.Step(); + SQLite3DB::Statement st=m_db->Prepare("UPDATE tblMessageInserts SET Day=?, InsertIndex=?, Inserted='true' WHERE MessageUUID=?;"); + st.Bind(0,idparts[6]); + st.Bind(1,index); + st.Bind(2,idparts[1]); + st.Step(); - // update the messageuuid to the real messageuuid - st=m_db->Prepare("SELECT MessageXML FROM tblMessageInserts WHERE MessageUUID=?;"); - st.Bind(0,idparts[1]); - st.Step(); - if(st.RowReturned()) - { - std::string xmldata=""; - st.ResultText(0,xmldata); - xml.ParseXML(xmldata); - xml.SetMessageID(idparts[4]); - - SQLite3DB::Statement st2=m_db->Prepare("UPDATE tblMessageInserts SET MessageUUID=?, MessageXML=? WHERE MessageUUID=?;"); - st2.Bind(0,idparts[4]); - st2.Bind(1,xml.GetXML()); - st2.Bind(2,idparts[1]); - st2.Step(); - - //update file insert MessageUUID as well - st2=m_db->Prepare("UPDATE tblFileInserts SET MessageUUID=? WHERE MessageUUID=?;"); - st2.Bind(0,idparts[4]); - st2.Bind(1,idparts[1]); - st2.Step(); - } + // insert record into temp table so MessageList will be inserted ASAP + date=Poco::Timestamp(); + st=m_db->Prepare("INSERT INTO tmpMessageListInsert(LocalIdentityID,Date) VALUES(?,?);"); + st.Bind(0,localidentityid); + st.Bind(1,Poco::DateTimeFormatter::format(date,"%Y-%m-%d")); + st.Step(); + + // update the messageuuid to the real messageuuid + st=m_db->Prepare("SELECT MessageXML FROM tblMessageInserts WHERE MessageUUID=?;"); + st.Bind(0,idparts[1]); + st.Step(); + if(st.RowReturned()) + { + std::string xmldata=""; + st.ResultText(0,xmldata); + xml.ParseXML(xmldata); + xml.SetMessageID(idparts[4]); + + SQLite3DB::Statement st2=m_db->Prepare("UPDATE tblMessageInserts SET MessageUUID=?, MessageXML=? WHERE MessageUUID=?;"); + st2.Bind(0,idparts[4]); + st2.Bind(1,xml.GetXML()); + st2.Bind(2,idparts[1]); + st2.Step(); + + //update file insert MessageUUID as well + st2=m_db->Prepare("UPDATE tblFileInserts SET MessageUUID=? WHERE MessageUUID=?;"); + st2.Bind(0,idparts[4]); + st2.Bind(1,idparts[1]); + st2.Step(); + } + + RemoveFromInsertList(idparts[1]); - RemoveFromInsertList(idparts[1]); + m_log->debug("MessageInserter::HandlePutSuccessful successfully inserted message "+message["Identifier"]); - m_log->debug("MessageInserter::HandlePutSuccessful successfully inserted message "+message["Identifier"]); + } + else + { + m_log->debug("MessageInserter::HandlePutSuccessful for editioned SSK "+message["Identifier"]); + } return true; } @@ -211,6 +232,16 @@ const bool MessageInserter::StartInsert(const std::string &messageuuid) m_fcp->Send(message); m_fcp->Send(std::vector(xml.begin(),xml.end())); + // test insert as editioned SSK + message.Clear(); + message.SetName("ClientPut"); + message["URI"]=privatekey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|Message|-"+indexstr; + message["Identifier"]=m_fcpuniquename+"|"+message["URI"]; + message["UploadFrom"]="direct"; + message["DataLength"]=xmlsizestr; + m_fcp->Send(message); + m_fcp->Send(std::vector(xml.begin(),xml.end())); + m_inserting.push_back(messageuuid); m_log->debug("MessageInserter::StartInsert started message insert "+message["URI"]); diff --git a/src/freenet/messagelistinserter.cpp b/src/freenet/messagelistinserter.cpp index 56e7013..eb943ff 100644 --- a/src/freenet/messagelistinserter.cpp +++ b/src/freenet/messagelistinserter.cpp @@ -22,6 +22,14 @@ MessageListInserter::MessageListInserter(SQLite3DB::DB *db, FCPv2::Connection *f void MessageListInserter::CheckForNeededInsert() { + + // more than 10 minutes trying to insert - restart + if(m_inserting.size()>0 && (m_laststartedinsert+Poco::Timespan(0,0,10,0,0)<=Poco::DateTime())) + { + m_log->error("MessageListInserter::CheckForNeededInsert more than 10 minutes have passed without success/failure. Clearing inserts."); + m_inserting.clear(); + } + // only do 1 insert at a time if(m_inserting.size()==0) { @@ -42,7 +50,8 @@ void MessageListInserter::CheckForNeededInsert() sql="SELECT tblLocalIdentity.LocalIdentityID "; sql+="FROM tblLocalIdentity INNER JOIN tblMessageInserts ON tblLocalIdentity.LocalIdentityID=tblMessageInserts.LocalIdentityID "; sql+="WHERE tblMessageInserts.Day>=? AND ((tblLocalIdentity.LastInsertedMessageList<=? OR tblLocalIdentity.LastInsertedMessageList IS NULL OR tblLocalIdentity.LastInsertedMessageList='') OR tblLocalIdentity.LocalIdentityID IN (SELECT LocalIdentityID FROM tmpMessageListInsert)) "; - sql+="GROUP BY tblLocalIdentity.LocalIdentityID;"; + sql+="GROUP BY tblLocalIdentity.LocalIdentityID "; + sql+="ORDER BY tblLocalIdentity.LastInsertedMessageList;"; SQLite3DB::Statement st=m_db->Prepare(sql); st.Bind(0,Poco::DateTimeFormatter::format(previous,"%Y-%m-%d")); @@ -204,9 +213,10 @@ const bool MessageListInserter::StartInsert(const long &localidentityid) st.Finalize(); - st=m_db->Prepare("SELECT MessageDate, MessageIndex, PublicKey, MessageID FROM tblMessage INNER JOIN tblIdentity ON tblMessage.IdentityID=tblIdentity.IdentityID WHERE MessageIndex IS NOT NULL ORDER BY MessageDate DESC, MessageTime DESC LIMIT 200;"); + st=m_db->Prepare("SELECT MessageDate, MessageIndex, PublicKey, MessageID FROM tblMessage INNER JOIN tblIdentity ON tblMessage.IdentityID=tblIdentity.IdentityID WHERE MessageIndex IS NOT NULL ORDER BY MessageDate DESC, MessageTime DESC LIMIT 175;"); SQLite3DB::Statement st2=m_db->Prepare("SELECT BoardName FROM tblBoard INNER JOIN tblMessageBoard ON tblBoard.BoardID=tblMessageBoard.BoardID WHERE tblMessageBoard.MessageID=?;"); st.Step(); + while(st.RowReturned()) { std::string day; @@ -280,10 +290,19 @@ const bool MessageListInserter::StartInsert(const long &localidentityid) m_inserting.push_back(localidentityid); m_lastinsertedxml[localidentityid]=xmlstr; + m_laststartedinsert=Poco::DateTime(); + return true; } else { + + // xml was the same one that we inserted 30 minutes ago, reset date so we don't continue checking every minute + st=m_db->Prepare("UPDATE tblLocalIdentity SET LastInsertedMessageList=? WHERE LocalIdentityID=?;"); + st.Bind(0,Poco::DateTimeFormatter::format(Poco::DateTime(),"%Y-%m-%d %H:%M:%S")); + st.Bind(1,localidentityid); + st.Step(); + return false; } diff --git a/src/freenet/messagelistrequester.cpp b/src/freenet/messagelistrequester.cpp index c5cb291..e54e0f2 100644 --- a/src/freenet/messagelistrequester.cpp +++ b/src/freenet/messagelistrequester.cpp @@ -1,7 +1,6 @@ #include "../../include/freenet/messagelistrequester.h" #include "../../include/freenet/messagelistxml.h" -#include #include #include #include @@ -73,41 +72,51 @@ const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &dates } } -void MessageListRequester::GetBoardList(std::map &boards) +void MessageListRequester::GetBoardList(std::map &boards, const bool forceload) { - SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;"); - st.Step(); - while(st.RowReturned()) + // only query database when forced, or an 30 minutes have passed since last query + if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime()) { - std::string boardname=""; - std::string tempval=""; - st.ResultText(0,boardname); - st.ResultText(1,tempval); - - if(tempval=="true") - { - boards[boardname]=true; - } - else + m_boardscache.clear(); + SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;"); + st.Step(); + while(st.RowReturned()) { - boards[boardname]=false; - } + std::string boardname=""; + std::string tempval=""; + st.ResultText(0,boardname); + st.ResultText(1,tempval); - st.Step(); + if(tempval=="true") + { + m_boardscache[boardname]=true; + } + else + { + m_boardscache[boardname]=false; + } + + st.Step(); + } + m_boardscacheupdate=Poco::DateTime(); } + + boards=m_boardscache; + } const bool MessageListRequester::HandleAllData(FCPv2::Message &message) { SQLite3DB::Statement st; - SQLite3DB::Statement trustst; std::vector idparts; long datalength; std::vector data; MessageListXML xml; long identityid; + long fromidentityid; long index; std::map boards; // list of boards and if we will save messages for that board or not + std::map identityids; // list of identity public keys and their id in the database bool addmessage=false; std::string boardsstr=""; std::string datestr=""; @@ -120,6 +129,8 @@ const bool MessageListRequester::HandleAllData(FCPv2::Message &message) StringFunctions::Convert(idparts[1],identityid); StringFunctions::Convert(idparts[2],index); + fromidentityid=identityid; + // wait for all data to be received from connection m_fcp->WaitForBytes(1000,datalength); @@ -138,9 +149,10 @@ const bool MessageListRequester::HandleAllData(FCPv2::Message &message) m_db->Execute("BEGIN;"); - SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;"); SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;"); - SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');"); + SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);"); + SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;"); + for(long i=0; iExecute("COMMIT;"); m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]); @@ -293,6 +332,12 @@ const bool MessageListRequester::HandleAllData(FCPv2::Message &message) // remove this identityid from request list RemoveFromRequestList(identityid); + // keep 2 days of request indexes in the cache + while(m_requestindexcache.size()>2) + { + m_requestindexcache.erase(m_requestindexcache.begin()); + } + return true; } @@ -383,6 +428,8 @@ void MessageListRequester::Initialize() option.Get("MessageDownloadMaxDaysBackward",tempval); StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward); + m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0); + } void MessageListRequester::PopulateIDList() @@ -396,11 +443,11 @@ void MessageListRequester::PopulateIDList() // select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) if(m_localtrustoverrides==false) { - st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); } else { - st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); + st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;"); } st.Step(); diff --git a/src/freenet/messagerequester.cpp b/src/freenet/messagerequester.cpp index f1429ab..043d2bf 100644 --- a/src/freenet/messagerequester.cpp +++ b/src/freenet/messagerequester.cpp @@ -274,6 +274,8 @@ const bool MessageRequester::HandleAllData(FCPv2::Message &message) //m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageRequester::HandleAddData could not insert message into database. "+message["Identifier"]); } + st.Finalize(); + m_db->Execute("COMMIT;"); } // if validmessage @@ -314,6 +316,14 @@ const bool MessageRequester::HandleGetFailed(FCPv2::Message &message) m_log->error("MessageRequester::HandleGetFailed fatal error requesting "+message["Identifier"]); } + // increase the failure count of the identity who gave us this index + st=m_db->Prepare("UPDATE tblIdentity SET FailureCount=FailureCount+1 WHERE IdentityID IN (SELECT FromIdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?);"); + st.Bind(0,identityid); + st.Bind(1,idparts[3]); + st.Bind(2,index); + st.Step(); + st.Finalize(); + // remove this identityid from request list RemoveFromRequestList(requestid); @@ -419,7 +429,7 @@ void MessageRequester::PopulateIDList() { sql+="AND (tblIdentity.LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (tblIdentity.LocalMessageTrust IS NULL AND (tblIdentity.PeerMessageTrust IS NULL OR tblIdentity.PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) "; } - sql+="AND tblIdentity.Name <> '' "; + sql+="AND tblIdentity.Name <> '' AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') "; // sort by day descending - in case there is a bunch of messages on a day that keep timing out, we will eventually get to the next day and hopefully find messages there // secondary ascending sort on tries // tertiary sort on request index (so we get low indexes first) @@ -503,7 +513,8 @@ void MessageRequester::StartRequest(const std::string &requestid) message["Identifier"]=m_fcpuniquename+"|"+requestid+"|"+parts[0]+"|"+parts[1]+"|"+parts[2]+"|"+message["URI"]; message["ReturnType"]="direct"; message["MaxSize"]="1000000"; // 1 MB - message["MaxRetries"]="-1"; // use ULPR since we are fairly sure message exists since the author says it does + // don't use ULPR - we wan't to know of failures ASAP so we can mark them as such + //message["MaxRetries"]="-1"; // use ULPR since we are fairly sure message exists since the author says it does m_fcp->Send(message); diff --git a/src/freenet/messagexml.cpp b/src/freenet/messagexml.cpp index b5c766c..9a1334e 100644 --- a/src/freenet/messagexml.cpp +++ b/src/freenet/messagexml.cpp @@ -64,7 +64,7 @@ std::string MessageXML::GetXML() } } - return GenerateXML(doc); + return GenerateXML(doc,false); } void MessageXML::Initialize() diff --git a/src/freenet/trustlistinserter.cpp b/src/freenet/trustlistinserter.cpp index 10bfbce..3461078 100644 --- a/src/freenet/trustlistinserter.cpp +++ b/src/freenet/trustlistinserter.cpp @@ -261,7 +261,8 @@ void TrustListInserter::StartInsert(const long localidentityid, const std::strin m_fcp->Send(message); m_fcp->Send(std::vector(data.begin(),data.end())); - // insert to USK + // insert to USK - not used, but don't remove code yet + /* message.Clear(); message.SetName("ClientPutComplexDir"); message["URI"]="USK"+privatekey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|TrustList/0/"; @@ -272,6 +273,7 @@ void TrustListInserter::StartInsert(const long localidentityid, const std::strin message["Files.0.DataLength"]=datasizestr; m_fcp->Send(message); m_fcp->Send(std::vector(data.begin(),data.end())); + */ m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='true' WHERE LocalIdentityID="+localidentityidstr+";"); diff --git a/src/freenet/trustlistrequester.cpp b/src/freenet/trustlistrequester.cpp index 96d24df..efede1d 100644 --- a/src/freenet/trustlistrequester.cpp +++ b/src/freenet/trustlistrequester.cpp @@ -294,7 +294,7 @@ void TrustListRequester::PopulateIDList() // select identities we want to query (we've seen them today and they are publishing trust list) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending) sql="SELECT IdentityID FROM tblIdentity "; - sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') )"; + sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') ) AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount')"; sql+="ORDER BY LocalTrustListTrust DESC, LastSeen;"; SQLite3DB::Statement st=m_db->Prepare(sql); diff --git a/src/http/pages/forummainpage.cpp b/src/http/pages/forummainpage.cpp index ae97874..5ef0e7a 100644 --- a/src/http/pages/forummainpage.cpp +++ b/src/http/pages/forummainpage.cpp @@ -14,7 +14,7 @@ const std::string ForumMainPage::GeneratePage(const std::string &method, const s content+="\r\n"; content+="\r\n"; - SQLite3DB::Statement newmessagesst=m_db->Prepare("SELECT tblMessage.MessageID FROM tblMessage INNER JOIN tblMessageBoard ON tblMessage.MessageID=tblMessageBoard.MessageID INNER JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID WHERE tblMessageBoard.BoardID=? AND tblMessage.Read=0 LIMIT 0,1;"); + SQLite3DB::Statement newmessagesst=m_db->Prepare("SELECT tblMessage.MessageID FROM tblMessage INNER JOIN tblMessageBoard ON tblMessage.MessageID=tblMessageBoard.MessageID INNER JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID WHERE tblMessageBoard.BoardID=? AND tblThread.BoardID=? AND tblMessage.Read=0 LIMIT 0,1;"); SQLite3DB::Statement lastmessagest=m_db->Prepare("SELECT tblMessage.MessageID, tblMessage.IdentityID, tblMessage.FromName, tblMessage.Subject, tblMessage.MessageDate || ' ' || tblMessage.MessageTime, tblThread.ThreadID FROM tblMessage INNER JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID WHERE tblThread.BoardID=? ORDER BY tblMessage.MessageDate || tblMessage.MessageTime DESC LIMIT 0,1;"); SQLite3DB::Statement st=m_db->Prepare("SELECT tblBoard.BoardID, BoardName, BoardDescription, COUNT(tblThreadPost.MessageID) FROM tblBoard LEFT JOIN tblThread ON tblBoard.BoardID=tblThread.BoardID LEFT JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE Forum='true' GROUP BY tblBoard.BoardID ORDER BY BoardName COLLATE NOCASE;"); @@ -37,6 +37,7 @@ const std::string ForumMainPage::GeneratePage(const std::string &method, const s content+="
NewForumPostsLast Post
"; newmessagesst.Bind(0,boardid); + newmessagesst.Bind(1,boardid); newmessagesst.Step(); if(newmessagesst.RowReturned()) { diff --git a/src/optionssetup.cpp b/src/optionssetup.cpp index b1ec99e..a802c16 100644 --- a/src/optionssetup.cpp +++ b/src/optionssetup.cpp @@ -395,6 +395,38 @@ void SetupDefaultOptions(SQLite3DB::DB *db) upd.Step(); upd.Reset(); + st.Bind(0,"MaxFailureCount"); + st.Bind(1,"1000"); + st.Step(); + st.Reset(); + upd.Bind(0,"Requests"); + upd.Bind(1,order++); + upd.Bind(2); + upd.Bind(3,"The maximum number of failed message requests an identity must accumulate before you will completely ignore an identity. Request failures can happen even under the best circumstances, and may accumulate rapidly, so it is best to keep this at a high level to avoid false positives."); + upd.Bind(4,"textbox"); + upd.Bind(5); + upd.Bind(6); + upd.Bind(7,"advanced"); + upd.Bind(8,"MaxFailureCount"); + upd.Step(); + upd.Reset(); + + st.Bind(0,"FailureCountReduction"); + st.Bind(1,"500"); + st.Step(); + st.Reset(); + upd.Bind(0,"Requests"); + upd.Bind(1,order++); + upd.Bind(2); + upd.Bind(3,"Each identity's failure count will be reduced by this amount every day."); + upd.Bind(4,"textbox"); + upd.Bind(5); + upd.Bind(6); + upd.Bind(7,"advanced"); + upd.Bind(8,"FailureCountReduction"); + upd.Step(); + upd.Reset(); + st.Bind(0,"MinLocalMessageTrust"); st.Bind(1,"50"); st.Step(); diff --git a/src/threadbuilder.cpp b/src/threadbuilder.cpp index 79102b9..7df5a54 100644 --- a/src/threadbuilder.cpp +++ b/src/threadbuilder.cpp @@ -3,6 +3,7 @@ #include "../include/dbsetup.h" #include "../include/stringfunctions.h" +#include "../include/option.h" const bool ThreadBuilder::Build(const long messageid, const long boardid, const bool bydate) { @@ -10,21 +11,44 @@ const bool ThreadBuilder::Build(const long messageid, const long boardid, const int threadid=-1; MessageThread mt(m_db); std::vector m_threadmessages; + std::string logmessage(""); // temp var to help track down exactly when corruption occurrs + std::string ll(""); + Option option(m_db); + + option.Get("LogLevel",ll); mt.Load(messageid,boardid,bydate); m_threadmessages=mt.GetNodes(); - m_db->Execute("BEGIN;"); + //m_db->Execute("BEGIN;"); // find threadid of this mesage if it already exists in a thread SQLite3DB::Statement st=m_db->Prepare("SELECT tblThread.ThreadID FROM tblThread INNER JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE tblThread.BoardID=? AND tblThreadPost.MessageID=?;"); st.Bind(0,boardid); st.Bind(1,messageid); + if(ll=="8") + { + std::string temp1(""); + std::string temp2(""); + StringFunctions::Convert(boardid,temp1); + StringFunctions::Convert(messageid,temp2); + + logmessage+="initial bound boardid=" + temp1 + " messageid=" + temp2 + " | "; + } + st.Step(); if(st.RowReturned()) { st.ResultInt(0,threadid); + + if(ll=="8") + { + std::string temp1(""); + StringFunctions::Convert(threadid,temp1); + + logmessage+="result threadid=" + temp1 + " | "; + } } else { @@ -36,9 +60,36 @@ const bool ThreadBuilder::Build(const long messageid, const long boardid, const st.Bind(1,(*i).m_messageid); st.Step(); + if(ll=="8") + { + std::string temp1(""); + std::string temp2(""); + StringFunctions::Convert(boardid,temp1); + StringFunctions::Convert((*i).m_messageid,temp2); + + logmessage+="find bound boardid=" + temp1 + " messageid=" + temp2 + " | "; + } + if(st.RowReturned()) { st.ResultInt(0,threadid); + if(ll=="8") + { + std::string temp1(""); + StringFunctions::Convert(threadid,temp1); + + logmessage+="find result threadid=" + temp1 + " | "; + } + } + else + { + if(ll=="8") + { + std::string temp1(""); + StringFunctions::Convert(threadid,temp1); + + logmessage+="find not found | "; + } } st.Reset(); @@ -52,6 +103,17 @@ const bool ThreadBuilder::Build(const long messageid, const long boardid, const st.Bind(0,boardid); st.Step(true); threadid=st.GetLastInsertRowID(); + + if(ll=="8") + { + std::string temp1(""); + std::string temp2(""); + StringFunctions::Convert(boardid,temp1); + StringFunctions::Convert(threadid,temp2); + + logmessage+="insert thread bind boardid=" + temp1 + " result threadid=" + temp2 + " | "; + } + } } @@ -63,10 +125,30 @@ const bool ThreadBuilder::Build(const long messageid, const long boardid, const st2.Bind(2,threadid); st2.Step(); + if(ll=="8") + { + std::string temp1(""); + std::string temp2(""); + std::string temp3(""); + StringFunctions::Convert(m_threadmessages[0].m_messageid,temp1); + StringFunctions::Convert(m_threadmessages[m_threadmessages.size()-1].m_messageid,temp2); + StringFunctions::Convert(threadid,temp3); + + logmessage+="update bind fmessageid=" + temp1 + " lmessageid=" + temp2 + " threadid=" + temp3 + " | "; + } + SQLite3DB::Statement st3=m_db->Prepare("DELETE FROM tblThreadPost WHERE ThreadID=?;"); st3.Bind(0,threadid); st3.Step(); + if(ll=="8") + { + std::string temp1(""); + StringFunctions::Convert(threadid,temp1); + + logmessage+="delete bind threadid=" + temp1 + " | "; + } + SQLite3DB::Statement deleteotherst=m_db->Prepare("DELETE FROM tblThread WHERE ThreadID IN (SELECT tblThread.ThreadID FROM tblThreadPost INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID WHERE tblThread.BoardID=? AND tblThreadPost.MessageID=?);"); count=0; @@ -78,11 +160,34 @@ const bool ThreadBuilder::Build(const long messageid, const long boardid, const deleteotherst.Step(); deleteotherst.Reset(); + if(ll=="8") + { + std::string temp1(""); + std::string temp2(""); + StringFunctions::Convert(boardid,temp1); + StringFunctions::Convert((*i).m_messageid,temp2); + + logmessage+="deleteother bind boardid=" + temp1 + " messageid=" + temp2 + " | "; + } + st4.Bind(0,threadid); st4.Bind(1,(*i).m_messageid); st4.Bind(2,count); st4.Step(); st4.Reset(); + + if(ll=="8") + { + std::string temp1(""); + std::string temp2(""); + std::string temp3(""); + StringFunctions::Convert(threadid,temp1); + StringFunctions::Convert((*i).m_messageid,temp2); + StringFunctions::Convert(count,temp3); + + logmessage+="insertthreadpost bind threadid=" + temp1 + " messageid=" + temp2 + " count=" + temp3 + " | "; + } + } } else @@ -91,10 +196,25 @@ const bool ThreadBuilder::Build(const long messageid, const long boardid, const st2.Bind(0,threadid); st2.Step(); + if(ll=="8") + { + std::string temp1(""); + StringFunctions::Convert(threadid,temp1); + + logmessage+="delete thread bind threadid=" + temp1 + " | "; + } + m_log->trace("ThreadBuilder::Build deleted thread"); } - m_db->Execute("COMMIT;"); + st.Finalize(); + + //m_db->Execute("COMMIT;"); + + if(ll=="8") + { + m_log->trace(logmessage); + } return true;