void ConvertDB0112To0113(SQLite3DB::DB *db);\r
void ConvertDB0113To0114(SQLite3DB::DB *db);\r
void ConvertDB0114To0115(SQLite3DB::DB *db);\r
+void ConvertDB0115To0116(SQLite3DB::DB *db);\r
\r
// TODO remove sometime after 0.1.17\r
void FixCapitalBoardNames(SQLite3DB::DB *db);\r
template <class IDTYPE>\r
void IIndexInserter<IDTYPE>::Process()\r
{\r
- Poco::DateTime now;\r
-\r
- if(m_lastchecked<(now-Poco::Timespan(0,0,1,0,0)))\r
+ if(m_lastchecked<(Poco::DateTime()-Poco::Timespan(0,0,1,0,0)))\r
{\r
CheckForNeededInsert();\r
- m_lastchecked=now;\r
+ m_lastchecked=Poco::DateTime();\r
}\r
}\r
\r
\r
long m_daysbackward;\r
std::map<long,std::string> m_lastinsertedxml; // last xml document inserted for each local identity\r
+ Poco::DateTime m_laststartedinsert;\r
\r
};\r
\r
#include "iindexrequester.h"\r
\r
#include <map>\r
+#include <set>\r
+\r
+#include <Poco/DateTime.h>\r
\r
class MessageListRequester:public IIndexRequester<long>\r
{\r
void StartRedirectRequest(FCPv2::Message &message);\r
const bool HandleAllData(FCPv2::Message &message);\r
const bool HandleGetFailed(FCPv2::Message &message);\r
- void GetBoardList(std::map<std::string,bool> &boards);\r
+ void GetBoardList(std::map<std::string,bool> &boards, const bool forceload=false);\r
const bool CheckDateNotFuture(const std::string &datestr) const;\r
const bool CheckDateWithinMaxDays(const std::string &datestr) const;\r
\r
bool m_savetonewboards;\r
long m_messagedownloadmaxdaysbackward;\r
\r
+ std::map<std::string,bool> m_boardscache;\r
+ Poco::DateTime m_boardscacheupdate; // last time we updated the boards cache\r
+\r
+ std::map<std::string,std::map<long,std::set<long> > > m_requestindexcache; // date - identity id - index\r
+\r
};\r
\r
#endif // _messagelistrequester_\r
\r
#define VERSION_MAJOR "0"\r
#define VERSION_MINOR "3"\r
-#define VERSION_RELEASE "29"\r
+#define VERSION_RELEASE "31"\r
#define FMS_VERSION VERSION_MAJOR"."VERSION_MINOR"."VERSION_RELEASE\r
-#define FMS_FREESITE_USK "USK@0npnMrqZNKRCRoGojZV93UNHCMN-6UU3rRSAmP6jNLE,~BG-edFtdCC1cSH4O3BWdeIYa8Sw5DfyrSV-TKdO5ec,AQACAAE/fms/91/"\r
-#define FMS_VERSION_EDITION "31"\r
+#define FMS_FREESITE_USK "USK@0npnMrqZNKRCRoGojZV93UNHCMN-6UU3rRSAmP6jNLE,~BG-edFtdCC1cSH4O3BWdeIYa8Sw5DfyrSV-TKdO5ec,AQACAAE/fms/93/"\r
+#define FMS_VERSION_EDITION "33"\r
\r
typedef Poco::ScopedLock<Poco::FastMutex> Guard;\r
\r
return returntext;\r
}\r
\r
- const std::string GenerateXML(Poco::AutoPtr<Poco::XML::Document> doc)\r
+ const std::string GenerateXML(Poco::AutoPtr<Poco::XML::Document> doc, const bool prettyprint=true)\r
{\r
std::ostringstream str;\r
if(doc)\r
{\r
Poco::XML::DOMWriter dr;\r
- dr.setOptions(Poco::XML::XMLWriter::WRITE_XML_DECLARATION | Poco::XML::XMLWriter::PRETTY_PRINT);\r
+ if(prettyprint==true)\r
+ {\r
+ dr.setOptions(Poco::XML::XMLWriter::WRITE_XML_DECLARATION | Poco::XML::XMLWriter::PRETTY_PRINT);\r
+ }\r
+ else\r
+ {\r
+ dr.setOptions(Poco::XML::XMLWriter::WRITE_XML_DECLARATION);\r
+ }\r
dr.setNewLine(Poco::XML::XMLWriter::NEWLINE_CRLF);\r
dr.writeNode(str,doc);\r
}\r
db->Execute("UPDATE tblDBVersion SET Major=1, Minor=15;");\r
}\r
\r
+void ConvertDB0115To0116(SQLite3DB::DB *db)\r
+{\r
+ // Add FromIdentityID to tblMessageRequests so we know who we got this index from\r
+ // Add FailureCount\r
+\r
+ db->Execute("ALTER TABLE tblMessageRequests ADD COLUMN FromIdentityID INTEGER;");\r
+ db->Execute("ALTER TABLE tblIdentity ADD COLUMN FailureCount INTEGER CHECK(FailureCount>=0) DEFAULT 0;");\r
+\r
+ db->Execute("UPDATE tblDBVersion SET Major=1, Minor=16;");\r
+}\r
+\r
void FixCapitalBoardNames(SQLite3DB::DB *db)\r
{\r
\r
{\r
std::string dbres=TestDBIntegrity(m_db);\r
m_log->trace("DBMaintenanceThread::Do10MinuteMaintenance() middle TestDBIntegrity returned "+dbres);\r
+ if(dbres!="ok")\r
+ {\r
+ m_db->Execute("REINDEX;");\r
+ dbres=TestDBIntegrity(m_db);\r
+ m_log->trace("DBMaintenanceThread::Do10MinuteMaintenenace() middle after reindex returned "+dbres);\r
+ }\r
}\r
\r
// now rebuild threads where the message has been deleted\r
st.Step();\r
}\r
\r
+ st.Finalize();\r
+ upd.Finalize();\r
+\r
// insert all identities not in trust list already\r
m_db->Execute("INSERT INTO tblIdentityTrust(LocalIdentityID,IdentityID) SELECT LocalIdentityID,IdentityID FROM tblLocalIdentity,tblIdentity WHERE LocalIdentityID || '_' || IdentityID NOT IN (SELECT LocalIdentityID || '_' || IdentityID FROM tblIdentityTrust);");\r
\r
st.Step();\r
}\r
\r
+ st.Finalize();\r
+ st2.Finalize();\r
+ upd.Finalize();\r
+\r
m_db->Execute("COMMIT;");\r
\r
m_log->debug("PeriodicDBMaintenance::Do6HourMaintenance");\r
// try to re-attach messages from identities that were previously deleted, but have been since re-added\r
// first get the names from messages that have a NULL IdentityID\r
SQLite3DB::Statement st=m_db->Prepare("SELECT FromName FROM tblMessage WHERE IdentityID IS NULL GROUP BY FromName;");\r
+ SQLite3DB::Statement findst=m_db->Prepare("SELECT IdentityID,PublicKey FROM tblIdentity WHERE Name=?;");\r
st.Step();\r
while(st.RowReturned())\r
{\r
}\r
\r
// find identities with this name\r
- SQLite3DB::Statement st2=m_db->Prepare("SELECT IdentityID,PublicKey FROM tblIdentity WHERE Name=?;");\r
- st2.Bind(0,namepart);\r
- st2.Step();\r
- while(st2.RowReturned())\r
+ findst.Bind(0,namepart);\r
+ findst.Step();\r
+ while(findst.RowReturned())\r
{\r
publickey="";\r
identityid=0;\r
- st2.ResultText(1,publickey);\r
+ findst.ResultText(1,publickey);\r
// check if public key matches 2nd part\r
if(parts.size()>1 && publickey.find(parts[1])==4)\r
{\r
// we have the identity - so update the messages table with the identityid\r
- st2.ResultInt(0,identityid);\r
+ findst.ResultInt(0,identityid);\r
\r
SQLite3DB::Statement st3=m_db->Prepare("UPDATE tblMessage SET IdentityID=? WHERE FromName=? AND IdentityID IS NULL;");\r
st3.Bind(0,identityid);\r
st3.Bind(1,name);\r
st3.Step();\r
}\r
- st2.Step();\r
+ findst.Step();\r
}\r
+ findst.Reset();\r
\r
st.Step();\r
}\r
m_db->Execute("DELETE FROM tblIdentityTrust WHERE LocalIdentityID NOT IN (SELECT LocalIdentityID FROM tblLocalIdentity);");\r
m_db->Execute("DELETE FROM tblIdentityTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);");\r
\r
+\r
+ // reduce failure count for each identity\r
+ m_db->Execute("UPDATE tblIdentity SET FailureCount=0 WHERE FailureCount<(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');");\r
+ m_db->Execute("UPDATE tblIdentity SET FailureCount=FailureCount-(SELECT OptionValue FROM tblOption WHERE OptionName='FailureCountReduction') WHERE FailureCount>=(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');");\r
+\r
+ st.Finalize();\r
+ findst.Finalize();\r
+\r
m_db->Execute("COMMIT;");\r
\r
m_log->debug("PeriodicDBMaintenance::Do1DayMaintenance");\r
major=1;\r
minor=15;\r
}\r
+ if(major==1 && minor==15)\r
+ {\r
+ ConvertDB0115To0116(db);\r
+ major=1;\r
+ minor=16;\r
+ }\r
}\r
else\r
{\r
- db->Execute("INSERT INTO tblDBVersion(Major,Minor) VALUES(1,15);");\r
+ db->Execute("INSERT INTO tblDBVersion(Major,Minor) VALUES(1,16);");\r
}\r
\r
- db->Execute("UPDATE tblDBVersion SET Major=1, Minor=15;");\r
+ db->Execute("UPDATE tblDBVersion SET Major=1, Minor=16;");\r
\r
db->Execute("CREATE TABLE IF NOT EXISTS tblFMSVersion(\\r
Major INTEGER,\\r
PeerTrustListTrust INTEGER CHECK(PeerTrustListTrust BETWEEN 0 AND 100) DEFAULT NULL,\\r
AddedMethod TEXT,\\r
Hidden BOOL CHECK(Hidden IN('true','false')) DEFAULT 'false',\\r
- PurgeDate DATETIME\\r
+ PurgeDate DATETIME,\\r
+ FailureCount INTEGER CHECK(FailureCount>=0) DEFAULT 0\\r
);");\r
\r
db->Execute("CREATE TABLE IF NOT EXISTS tblIdentityRequests(\\r
FromMessageList BOOL CHECK(FromMessageList IN('true','false')) DEFAULT 'false',\\r
Found BOOL CHECK(Found IN('true','false')) DEFAULT 'false',\\r
Tries INTEGER DEFAULT 0,\\r
- Key TEXT\\r
+ Key TEXT,\\r
+ FromIdentityID INTEGER\\r
);");\r
\r
db->Execute("CREATE UNIQUE INDEX IF NOT EXISTS idxMessageRequest ON tblMessageRequests(IdentityID,Day,RequestIndex);");\r
\r
if(m_localtrustoverrides==false)\r
{\r
- st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND PublishBoardList='true' ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND PublishBoardList='true' AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
}\r
else\r
{\r
- st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND PublishBoardList='true' ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(today,"%Y-%m-%d")+"' AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND PublishBoardList='true' AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
}\r
st.Step();\r
\r
int index;\r
std::string identityidstr;\r
\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE identityid=?;");\r
+ SQLite3DB::Statement st=m_db->Prepare("SELECT PublicKey FROM tblIdentity WHERE IdentityID=?;");\r
st.Bind(0,identityid);\r
st.Step();\r
\r
Poco::DateTime lastdate;\r
int tzdiff=0;\r
Poco::DateTimeParser::tryParse("%Y-%m-%d",idparts[4],lastdate,tzdiff);\r
-\r
- if(lastdate.day()==now.day())\r
+ \r
+ // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now\r
+ if(message["Identifier"].find(".xml")!=std::string::npos)\r
{\r
- m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false', LastInsertedIdentity='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";");\r
+ if(lastdate.day()==now.day())\r
+ {\r
+ m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false', LastInsertedIdentity='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S")+"' WHERE LocalIdentityID="+idparts[1]+";");\r
+ }\r
+ else\r
+ {\r
+ m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";");\r
+ }\r
+ m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");");\r
+ m_log->debug("IdentityInserter::HandleMessage inserted Identity xml");\r
}\r
else\r
{\r
- m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";");\r
+ m_log->trace("IdentityInserter::HandleMessage inserted editioned Identity xml");\r
}\r
- m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");");\r
- m_log->debug("IdentityInserter::HandleMessage inserted Identity xml");\r
return true;\r
}\r
\r
if(message.GetName()=="PutFailed")\r
{\r
- m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";");\r
- m_log->debug("IdentityInserter::HandleMessage failure inserting Identity xml. Code="+message["Code"]+" Description="+message["CodeDescription"]);\r
- \r
- // if code 9 (collision), then insert index into inserted table\r
- if(message["Code"]=="9")\r
+ // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now\r
+ if(message["Identifier"].find(".xml")!=std::string::npos)\r
{\r
- m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");");\r
+ m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='false' WHERE LocalIdentityID="+idparts[1]+";");\r
+ m_log->debug("IdentityInserter::HandleMessage failure inserting Identity xml. Code="+message["Code"]+" Description="+message["CodeDescription"]);\r
+ \r
+ // if code 9 (collision), then insert index into inserted table\r
+ if(message["Code"]=="9")\r
+ {\r
+ m_db->Execute("INSERT INTO tblLocalIdentityInserts(LocalIdentityID,Day,InsertIndex) VALUES("+idparts[1]+",'"+idparts[4]+"',"+idparts[2]+");");\r
+ }\r
+ }\r
+ else\r
+ {\r
+ m_log->trace("IdentityInserter::HandleMessage PutFailed for editioned SSK error code "+message["Code"]+ " id "+message["Identifier"]);\r
}\r
\r
return true;\r
m_fcp->Send(mess);\r
m_fcp->Send(std::vector<char>(data.begin(),data.end()));\r
\r
+ // test insert as editioned SSK\r
+ mess.Clear();\r
+ mess.SetName("ClientPut");\r
+ mess["URI"]=privatekey+messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|Identity|-"+indexstr;\r
+ mess["Identifier"]="IdentityInserter|"+mess["URI"];\r
+ mess["UploadFrom"]="direct";\r
+ mess["DataLength"]=datasizestr;\r
+ m_fcp->Send(mess);\r
+ m_fcp->Send(std::vector<char>(data.begin(),data.end()));\r
+\r
m_db->Execute("UPDATE tblLocalIdentity SET InsertingIdentity='true' WHERE LocalIdentityID="+idstring+";");\r
\r
}\r
date.assign(date.year(),date.month(),date.day(),0,0,0);\r
\r
// select identities we want to query (haven't seen yet today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen IS NOT NULL AND LastSeen<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S")+"' ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen IS NOT NULL AND LastSeen<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S")+"' AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
st.Step();\r
\r
m_ids.clear();\r
m_fcp->Send(message);\r
m_fcp->Send(std::vector<char>(xmldata.begin(),xmldata.end()));\r
\r
- // insert to USK\r
+ // insert to USK - not used, but don't remove code yet\r
+ /*\r
message.Clear();\r
message.SetName("ClientPutComplexDir");\r
message["URI"]="USK"+privatekey.substr(3)+messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|IntroductionPuzzle/0/";\r
message["Files.0.DataLength"]=xmldatasizestr;\r
m_fcp->Send(message);\r
m_fcp->Send(std::vector<char>(xmldata.begin(),xmldata.end()));\r
+ */\r
\r
m_db->Execute("INSERT INTO tblIntroductionPuzzleInserts(UUID,Type,MimeType,LocalIdentityID,PuzzleData,PuzzleSolution) VALUES('"+xml.GetUUID()+"','captcha','image/bmp',"+idstring+",'"+encodedpuzzle+"','"+solutionstring+"');");\r
\r
st.Finalize();\r
\r
// select identities that aren't single use, are publishing a trust list, and have been seen today ( order by trust DESC and limit to limitnum )\r
- st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublishTrustList='true' AND PublicKey IS NOT NULL AND PublicKey <> '' AND SingleUse='false' AND (LocalTrustListTrust IS NULL OR LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust')) AND LastSeen>='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"' ORDER BY LocalMessageTrust DESC LIMIT 0,"+limitnum+";");\r
+ st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublishTrustList='true' AND PublicKey IS NOT NULL AND PublicKey <> '' AND SingleUse='false' AND (LocalTrustListTrust IS NULL OR LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust')) AND LastSeen>='"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"' AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust DESC LIMIT 0,"+limitnum+";");\r
st.Step();\r
\r
m_ids.clear();\r
int index;\r
int localidentityid;\r
std::vector<std::string> idparts;\r
- StringFunctions::Split(message["Identifier"],"|",idparts);\r
- StringFunctions::Convert(idparts[2],localidentityid);\r
- StringFunctions::Convert(idparts[3],index);\r
\r
- // fatal put - or data exists - insert bogus index into database so we'll try to insert this message again\r
- if(message["Fatal"]=="true" || message["Code"]=="9")\r
+ // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now\r
+ if(message["Identifier"].find(".xml")!=std::string::npos)\r
{\r
- SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'true');");\r
- st.Bind(0,localidentityid);\r
- st.Bind(1,idparts[6]);\r
- st.Bind(2,index);\r
- st.Step();\r
- }\r
\r
- m_log->trace("MessageInserter::HandlePutFailed error code "+message["Code"]+" fatal="+message["Fatal"]);\r
+ StringFunctions::Split(message["Identifier"],"|",idparts);\r
+ StringFunctions::Convert(idparts[2],localidentityid);\r
+ StringFunctions::Convert(idparts[3],index);\r
\r
- RemoveFromInsertList(idparts[1]);\r
+ // fatal put - or data exists - insert bogus index into database so we'll try to insert this message again\r
+ if(message["Fatal"]=="true" || message["Code"]=="9")\r
+ {\r
+ SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'true');");\r
+ st.Bind(0,localidentityid);\r
+ st.Bind(1,idparts[6]);\r
+ st.Bind(2,index);\r
+ st.Step();\r
+ }\r
+\r
+ m_log->trace("MessageInserter::HandlePutFailed error code "+message["Code"]+" fatal="+message["Fatal"]);\r
+\r
+ RemoveFromInsertList(idparts[1]);\r
+\r
+ }\r
+ else\r
+ {\r
+ m_log->trace("MessageInserter::HandlePutFailed for editioned SSK error code "+message["Code"]+ " id "+message["Identifier"]);\r
+ }\r
\r
return true;\r
}\r
int index;\r
std::vector<std::string> idparts;\r
\r
- StringFunctions::Split(message["Identifier"],"|",idparts);\r
- StringFunctions::Convert(idparts[3],index);\r
- StringFunctions::Convert(idparts[2],localidentityid);\r
+ // do check to make sure this is the non-editioned SSK - we ignore failure/success for editioned SSK for now\r
+ if(message["Identifier"].find(".xml")!=std::string::npos)\r
+ {\r
\r
- SQLite3DB::Statement st=m_db->Prepare("UPDATE tblMessageInserts SET Day=?, InsertIndex=?, Inserted='true' WHERE MessageUUID=?;");\r
- st.Bind(0,idparts[6]);\r
- st.Bind(1,index);\r
- st.Bind(2,idparts[1]);\r
- st.Step();\r
+ StringFunctions::Split(message["Identifier"],"|",idparts);\r
+ StringFunctions::Convert(idparts[3],index);\r
+ StringFunctions::Convert(idparts[2],localidentityid);\r
\r
- // insert record into temp table so MessageList will be inserted ASAP\r
- date=Poco::Timestamp();\r
- st=m_db->Prepare("INSERT INTO tmpMessageListInsert(LocalIdentityID,Date) VALUES(?,?);");\r
- st.Bind(0,localidentityid);\r
- st.Bind(1,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
- st.Step();\r
+ SQLite3DB::Statement st=m_db->Prepare("UPDATE tblMessageInserts SET Day=?, InsertIndex=?, Inserted='true' WHERE MessageUUID=?;");\r
+ st.Bind(0,idparts[6]);\r
+ st.Bind(1,index);\r
+ st.Bind(2,idparts[1]);\r
+ st.Step();\r
\r
- // update the messageuuid to the real messageuuid\r
- st=m_db->Prepare("SELECT MessageXML FROM tblMessageInserts WHERE MessageUUID=?;");\r
- st.Bind(0,idparts[1]);\r
- st.Step();\r
- if(st.RowReturned())\r
- {\r
- std::string xmldata="";\r
- st.ResultText(0,xmldata);\r
- xml.ParseXML(xmldata);\r
- xml.SetMessageID(idparts[4]);\r
-\r
- SQLite3DB::Statement st2=m_db->Prepare("UPDATE tblMessageInserts SET MessageUUID=?, MessageXML=? WHERE MessageUUID=?;");\r
- st2.Bind(0,idparts[4]);\r
- st2.Bind(1,xml.GetXML());\r
- st2.Bind(2,idparts[1]);\r
- st2.Step();\r
-\r
- //update file insert MessageUUID as well\r
- st2=m_db->Prepare("UPDATE tblFileInserts SET MessageUUID=? WHERE MessageUUID=?;");\r
- st2.Bind(0,idparts[4]);\r
- st2.Bind(1,idparts[1]);\r
- st2.Step();\r
- }\r
+ // insert record into temp table so MessageList will be inserted ASAP\r
+ date=Poco::Timestamp();\r
+ st=m_db->Prepare("INSERT INTO tmpMessageListInsert(LocalIdentityID,Date) VALUES(?,?);");\r
+ st.Bind(0,localidentityid);\r
+ st.Bind(1,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
+ st.Step();\r
+\r
+ // update the messageuuid to the real messageuuid\r
+ st=m_db->Prepare("SELECT MessageXML FROM tblMessageInserts WHERE MessageUUID=?;");\r
+ st.Bind(0,idparts[1]);\r
+ st.Step();\r
+ if(st.RowReturned())\r
+ {\r
+ std::string xmldata="";\r
+ st.ResultText(0,xmldata);\r
+ xml.ParseXML(xmldata);\r
+ xml.SetMessageID(idparts[4]);\r
+\r
+ SQLite3DB::Statement st2=m_db->Prepare("UPDATE tblMessageInserts SET MessageUUID=?, MessageXML=? WHERE MessageUUID=?;");\r
+ st2.Bind(0,idparts[4]);\r
+ st2.Bind(1,xml.GetXML());\r
+ st2.Bind(2,idparts[1]);\r
+ st2.Step();\r
+\r
+ //update file insert MessageUUID as well\r
+ st2=m_db->Prepare("UPDATE tblFileInserts SET MessageUUID=? WHERE MessageUUID=?;");\r
+ st2.Bind(0,idparts[4]);\r
+ st2.Bind(1,idparts[1]);\r
+ st2.Step();\r
+ }\r
+\r
+ RemoveFromInsertList(idparts[1]);\r
\r
- RemoveFromInsertList(idparts[1]);\r
+ m_log->debug("MessageInserter::HandlePutSuccessful successfully inserted message "+message["Identifier"]);\r
\r
- m_log->debug("MessageInserter::HandlePutSuccessful successfully inserted message "+message["Identifier"]);\r
+ }\r
+ else\r
+ {\r
+ m_log->debug("MessageInserter::HandlePutSuccessful for editioned SSK "+message["Identifier"]);\r
+ }\r
\r
return true;\r
}\r
m_fcp->Send(message);\r
m_fcp->Send(std::vector<char>(xml.begin(),xml.end()));\r
\r
+ // test insert as editioned SSK\r
+ message.Clear();\r
+ message.SetName("ClientPut");\r
+ message["URI"]=privatekey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|Message|-"+indexstr;\r
+ message["Identifier"]=m_fcpuniquename+"|"+message["URI"];\r
+ message["UploadFrom"]="direct";\r
+ message["DataLength"]=xmlsizestr;\r
+ m_fcp->Send(message);\r
+ m_fcp->Send(std::vector<char>(xml.begin(),xml.end()));\r
+\r
m_inserting.push_back(messageuuid);\r
\r
m_log->debug("MessageInserter::StartInsert started message insert "+message["URI"]);\r
\r
void MessageListInserter::CheckForNeededInsert()\r
{\r
+\r
+ // more than 10 minutes trying to insert - restart\r
+ if(m_inserting.size()>0 && (m_laststartedinsert+Poco::Timespan(0,0,10,0,0)<=Poco::DateTime()))\r
+ {\r
+ m_log->error("MessageListInserter::CheckForNeededInsert more than 10 minutes have passed without success/failure. Clearing inserts.");\r
+ m_inserting.clear();\r
+ }\r
+\r
// only do 1 insert at a time\r
if(m_inserting.size()==0)\r
{\r
sql="SELECT tblLocalIdentity.LocalIdentityID ";\r
sql+="FROM tblLocalIdentity INNER JOIN tblMessageInserts ON tblLocalIdentity.LocalIdentityID=tblMessageInserts.LocalIdentityID ";\r
sql+="WHERE tblMessageInserts.Day>=? AND ((tblLocalIdentity.LastInsertedMessageList<=? OR tblLocalIdentity.LastInsertedMessageList IS NULL OR tblLocalIdentity.LastInsertedMessageList='') OR tblLocalIdentity.LocalIdentityID IN (SELECT LocalIdentityID FROM tmpMessageListInsert)) ";\r
- sql+="GROUP BY tblLocalIdentity.LocalIdentityID;";\r
+ sql+="GROUP BY tblLocalIdentity.LocalIdentityID ";\r
+ sql+="ORDER BY tblLocalIdentity.LastInsertedMessageList;";\r
\r
SQLite3DB::Statement st=m_db->Prepare(sql);\r
st.Bind(0,Poco::DateTimeFormatter::format(previous,"%Y-%m-%d"));\r
st.Finalize();\r
\r
\r
- st=m_db->Prepare("SELECT MessageDate, MessageIndex, PublicKey, MessageID FROM tblMessage INNER JOIN tblIdentity ON tblMessage.IdentityID=tblIdentity.IdentityID WHERE MessageIndex IS NOT NULL ORDER BY MessageDate DESC, MessageTime DESC LIMIT 200;");\r
+ st=m_db->Prepare("SELECT MessageDate, MessageIndex, PublicKey, MessageID FROM tblMessage INNER JOIN tblIdentity ON tblMessage.IdentityID=tblIdentity.IdentityID WHERE MessageIndex IS NOT NULL ORDER BY MessageDate DESC, MessageTime DESC LIMIT 175;");\r
SQLite3DB::Statement st2=m_db->Prepare("SELECT BoardName FROM tblBoard INNER JOIN tblMessageBoard ON tblBoard.BoardID=tblMessageBoard.BoardID WHERE tblMessageBoard.MessageID=?;");\r
st.Step();\r
+\r
while(st.RowReturned())\r
{\r
std::string day;\r
m_inserting.push_back(localidentityid);\r
m_lastinsertedxml[localidentityid]=xmlstr;\r
\r
+ m_laststartedinsert=Poco::DateTime();\r
+\r
return true;\r
}\r
else\r
{\r
+\r
+ // xml was the same one that we inserted 30 minutes ago, reset date so we don't continue checking every minute\r
+ st=m_db->Prepare("UPDATE tblLocalIdentity SET LastInsertedMessageList=? WHERE LocalIdentityID=?;");\r
+ st.Bind(0,Poco::DateTimeFormatter::format(Poco::DateTime(),"%Y-%m-%d %H:%M:%S"));\r
+ st.Bind(1,localidentityid);\r
+ st.Step();\r
+\r
return false;\r
}\r
\r
#include "../../include/freenet/messagelistrequester.h"\r
#include "../../include/freenet/messagelistxml.h"\r
\r
-#include <Poco/DateTime.h>\r
#include <Poco/DateTimeFormatter.h>\r
#include <Poco/DateTimeParser.h>\r
#include <Poco/Timestamp.h>\r
}\r
}\r
\r
-void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)\r
+void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards, const bool forceload)\r
{\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
- st.Step();\r
- while(st.RowReturned())\r
+ // only query database when forced, or an 30 minutes have passed since last query\r
+ if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime())\r
{\r
- std::string boardname="";\r
- std::string tempval="";\r
- st.ResultText(0,boardname);\r
- st.ResultText(1,tempval);\r
-\r
- if(tempval=="true")\r
- {\r
- boards[boardname]=true;\r
- }\r
- else\r
+ m_boardscache.clear();\r
+ SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
+ st.Step();\r
+ while(st.RowReturned())\r
{\r
- boards[boardname]=false;\r
- }\r
+ std::string boardname="";\r
+ std::string tempval="";\r
+ st.ResultText(0,boardname);\r
+ st.ResultText(1,tempval);\r
\r
- st.Step();\r
+ if(tempval=="true")\r
+ {\r
+ m_boardscache[boardname]=true;\r
+ }\r
+ else\r
+ {\r
+ m_boardscache[boardname]=false;\r
+ }\r
+\r
+ st.Step();\r
+ }\r
+ m_boardscacheupdate=Poco::DateTime();\r
}\r
+\r
+ boards=m_boardscache;\r
+\r
}\r
\r
const bool MessageListRequester::HandleAllData(FCPv2::Message &message)\r
{ \r
SQLite3DB::Statement st;\r
- SQLite3DB::Statement trustst;\r
std::vector<std::string> idparts;\r
long datalength;\r
std::vector<char> data;\r
MessageListXML xml;\r
long identityid;\r
+ long fromidentityid;\r
long index;\r
std::map<std::string,bool> boards; // list of boards and if we will save messages for that board or not\r
+ std::map<std::string,long> identityids; // list of identity public keys and their id in the database\r
bool addmessage=false;\r
std::string boardsstr="";\r
std::string datestr="";\r
StringFunctions::Convert(idparts[1],identityid);\r
StringFunctions::Convert(idparts[2],index);\r
\r
+ fromidentityid=identityid;\r
+\r
// wait for all data to be received from connection\r
m_fcp->WaitForBytes(1000,datalength);\r
\r
\r
m_db->Execute("BEGIN;");\r
\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
- SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");\r
+ SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);");\r
+ SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
+\r
for(long i=0; i<xml.MessageCount(); i++)\r
{\r
\r
\r
if(addmessage==true)\r
{\r
- st.Bind(0,identityid);\r
- st.Bind(1,xml.GetDate(i));\r
- st.Bind(2,xml.GetIndex(i));\r
- st.Step();\r
- if(st.RowReturned()==false)\r
- {\r
- mst.Bind(0,identityid);\r
- mst.Bind(1,xml.GetDate(i));\r
- mst.Bind(2,xml.GetIndex(i));\r
- mst.Step();\r
- mst.Reset();\r
- }\r
- st.Reset();\r
+ mst.Bind(0,identityid);\r
+ mst.Bind(1,xml.GetDate(i));\r
+ mst.Bind(2,xml.GetIndex(i));\r
+ mst.Bind(3,identityid);\r
+ mst.Step();\r
+ mst.Reset();\r
+\r
+ // We need to update ID here, in case this index was already inserted from another\r
+ // identity's message list. This doesn't reset try count - maybe we should if the from\r
+ // identity was another identity\r
+ ust.Bind(0,identityid);\r
+ ust.Bind(1,identityid);\r
+ ust.Bind(2,xml.GetDate(i));\r
+ ust.Bind(3,xml.GetIndex(i));\r
+ ust.Step();\r
+ ust.Reset();\r
+\r
+ m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i));\r
+\r
}\r
else\r
{\r
\r
if(addmessage==true)\r
{\r
- spk.Bind(0,xml.GetExternalIdentity(i));\r
- spk.Step();\r
- if(spk.RowReturned())\r
+ int thisidentityid=0;\r
+ if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end())\r
+ {\r
+ thisidentityid=identityids[xml.GetExternalIdentity(i)];\r
+ }\r
+ else\r
+ {\r
+ spk.Bind(0,xml.GetExternalIdentity(i));\r
+ spk.Step();\r
+\r
+ if(spk.RowReturned())\r
+ {\r
+ spk.ResultInt(0,thisidentityid);\r
+ identityids[xml.GetExternalIdentity(i)]=thisidentityid;\r
+ }\r
+\r
+ spk.Reset();\r
+ }\r
+\r
+ if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end())\r
{\r
- int thisidentityid=0;\r
- spk.ResultInt(0,thisidentityid);\r
mst.Bind(0,thisidentityid);\r
mst.Bind(1,xml.GetExternalDate(i));\r
mst.Bind(2,xml.GetExternalIndex(i));\r
+ mst.Bind(3,fromidentityid);\r
mst.Step();\r
mst.Reset();\r
+\r
+ m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i));\r
}\r
- spk.Reset();\r
}\r
else\r
{\r
st.Step();\r
st.Finalize();\r
\r
+ spk.Finalize();\r
+ mst.Finalize();\r
+ ust.Finalize();\r
+\r
m_db->Execute("COMMIT;");\r
\r
m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
// remove this identityid from request list\r
RemoveFromRequestList(identityid);\r
\r
+ // keep 2 days of request indexes in the cache\r
+ while(m_requestindexcache.size()>2)\r
+ {\r
+ m_requestindexcache.erase(m_requestindexcache.begin());\r
+ }\r
+\r
return true;\r
\r
}\r
option.Get("MessageDownloadMaxDaysBackward",tempval);\r
StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
\r
+ m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0);\r
+\r
}\r
\r
void MessageListRequester::PopulateIDList()\r
// select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
if(m_localtrustoverrides==false)\r
{\r
- st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
}\r
else\r
{\r
- st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
}\r
st.Step();\r
\r
//m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageRequester::HandleAddData could not insert message into database. "+message["Identifier"]);\r
}\r
\r
+ st.Finalize();\r
+\r
m_db->Execute("COMMIT;");\r
\r
} // if validmessage\r
m_log->error("MessageRequester::HandleGetFailed fatal error requesting "+message["Identifier"]);\r
}\r
\r
+ // increase the failure count of the identity who gave us this index\r
+ st=m_db->Prepare("UPDATE tblIdentity SET FailureCount=FailureCount+1 WHERE IdentityID IN (SELECT FromIdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?);");\r
+ st.Bind(0,identityid);\r
+ st.Bind(1,idparts[3]);\r
+ st.Bind(2,index);\r
+ st.Step();\r
+ st.Finalize();\r
+\r
// remove this identityid from request list\r
RemoveFromRequestList(requestid);\r
\r
{\r
sql+="AND (tblIdentity.LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (tblIdentity.LocalMessageTrust IS NULL AND (tblIdentity.PeerMessageTrust IS NULL OR tblIdentity.PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ";\r
}\r
- sql+="AND tblIdentity.Name <> '' ";\r
+ sql+="AND tblIdentity.Name <> '' AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ";\r
// sort by day descending - in case there is a bunch of messages on a day that keep timing out, we will eventually get to the next day and hopefully find messages there\r
// secondary ascending sort on tries\r
// tertiary sort on request index (so we get low indexes first)\r
message["Identifier"]=m_fcpuniquename+"|"+requestid+"|"+parts[0]+"|"+parts[1]+"|"+parts[2]+"|"+message["URI"];\r
message["ReturnType"]="direct";\r
message["MaxSize"]="1000000"; // 1 MB\r
- message["MaxRetries"]="-1"; // use ULPR since we are fairly sure message exists since the author says it does\r
+ // don't use ULPR - we wan't to know of failures ASAP so we can mark them as such\r
+ //message["MaxRetries"]="-1"; // use ULPR since we are fairly sure message exists since the author says it does\r
\r
m_fcp->Send(message);\r
\r
}\r
}\r
\r
- return GenerateXML(doc);\r
+ return GenerateXML(doc,false);\r
}\r
\r
void MessageXML::Initialize()\r
m_fcp->Send(message);\r
m_fcp->Send(std::vector<char>(data.begin(),data.end()));\r
\r
- // insert to USK\r
+ // insert to USK - not used, but don't remove code yet\r
+ /*\r
message.Clear();\r
message.SetName("ClientPutComplexDir");\r
message["URI"]="USK"+privatekey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|TrustList/0/";\r
message["Files.0.DataLength"]=datasizestr;\r
m_fcp->Send(message);\r
m_fcp->Send(std::vector<char>(data.begin(),data.end()));\r
+ */\r
\r
m_db->Execute("UPDATE tblLocalIdentity SET InsertingTrustList='true' WHERE LocalIdentityID="+localidentityidstr+";");\r
\r
\r
// select identities we want to query (we've seen them today and they are publishing trust list) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
sql="SELECT IdentityID FROM tblIdentity ";\r
- sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') )";\r
+ sql+="WHERE Name IS NOT NULL AND Name <> '' AND PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND PublishTrustList='true' AND LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust') AND ( PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust') ) AND tblIdentity.FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount')";\r
sql+="ORDER BY LocalTrustListTrust DESC, LastSeen;";\r
\r
SQLite3DB::Statement st=m_db->Prepare(sql);\r
content+="<table class=\"foruminfo\">\r\n";\r
content+="<thead><tr><th>New</th><th>Forum</th><th>Posts</th><th>Last Post</th></tr></thead>\r\n";\r
\r
- SQLite3DB::Statement newmessagesst=m_db->Prepare("SELECT tblMessage.MessageID FROM tblMessage INNER JOIN tblMessageBoard ON tblMessage.MessageID=tblMessageBoard.MessageID INNER JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID WHERE tblMessageBoard.BoardID=? AND tblMessage.Read=0 LIMIT 0,1;");\r
+ SQLite3DB::Statement newmessagesst=m_db->Prepare("SELECT tblMessage.MessageID FROM tblMessage INNER JOIN tblMessageBoard ON tblMessage.MessageID=tblMessageBoard.MessageID INNER JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID WHERE tblMessageBoard.BoardID=? AND tblThread.BoardID=? AND tblMessage.Read=0 LIMIT 0,1;");\r
SQLite3DB::Statement lastmessagest=m_db->Prepare("SELECT tblMessage.MessageID, tblMessage.IdentityID, tblMessage.FromName, tblMessage.Subject, tblMessage.MessageDate || ' ' || tblMessage.MessageTime, tblThread.ThreadID FROM tblMessage INNER JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID WHERE tblThread.BoardID=? ORDER BY tblMessage.MessageDate || tblMessage.MessageTime DESC LIMIT 0,1;");\r
\r
SQLite3DB::Statement st=m_db->Prepare("SELECT tblBoard.BoardID, BoardName, BoardDescription, COUNT(tblThreadPost.MessageID) FROM tblBoard LEFT JOIN tblThread ON tblBoard.BoardID=tblThread.BoardID LEFT JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE Forum='true' GROUP BY tblBoard.BoardID ORDER BY BoardName COLLATE NOCASE;");\r
content+="<td class=\"newposts\">";\r
\r
newmessagesst.Bind(0,boardid);\r
+ newmessagesst.Bind(1,boardid);\r
newmessagesst.Step();\r
if(newmessagesst.RowReturned())\r
{\r
upd.Step();\r
upd.Reset();\r
\r
+ st.Bind(0,"MaxFailureCount");\r
+ st.Bind(1,"1000");\r
+ st.Step();\r
+ st.Reset();\r
+ upd.Bind(0,"Requests");\r
+ upd.Bind(1,order++);\r
+ upd.Bind(2);\r
+ upd.Bind(3,"The maximum number of failed message requests an identity must accumulate before you will completely ignore an identity. Request failures can happen even under the best circumstances, and may accumulate rapidly, so it is best to keep this at a high level to avoid false positives.");\r
+ upd.Bind(4,"textbox");\r
+ upd.Bind(5);\r
+ upd.Bind(6);\r
+ upd.Bind(7,"advanced");\r
+ upd.Bind(8,"MaxFailureCount");\r
+ upd.Step();\r
+ upd.Reset();\r
+\r
+ st.Bind(0,"FailureCountReduction");\r
+ st.Bind(1,"500");\r
+ st.Step();\r
+ st.Reset();\r
+ upd.Bind(0,"Requests");\r
+ upd.Bind(1,order++);\r
+ upd.Bind(2);\r
+ upd.Bind(3,"Each identity's failure count will be reduced by this amount every day.");\r
+ upd.Bind(4,"textbox");\r
+ upd.Bind(5);\r
+ upd.Bind(6);\r
+ upd.Bind(7,"advanced");\r
+ upd.Bind(8,"FailureCountReduction");\r
+ upd.Step();\r
+ upd.Reset();\r
+\r
st.Bind(0,"MinLocalMessageTrust");\r
st.Bind(1,"50");\r
st.Step();\r
\r
#include "../include/dbsetup.h"\r
#include "../include/stringfunctions.h"\r
+#include "../include/option.h"\r
\r
const bool ThreadBuilder::Build(const long messageid, const long boardid, const bool bydate)\r
{\r
int threadid=-1;\r
MessageThread mt(m_db);\r
std::vector<MessageThread::threadnode> m_threadmessages;\r
+ std::string logmessage(""); // temp var to help track down exactly when corruption occurrs\r
+ std::string ll("");\r
+ Option option(m_db);\r
+\r
+ option.Get("LogLevel",ll);\r
\r
mt.Load(messageid,boardid,bydate);\r
m_threadmessages=mt.GetNodes();\r
\r
- m_db->Execute("BEGIN;");\r
+ //m_db->Execute("BEGIN;");\r
\r
// find threadid of this mesage if it already exists in a thread\r
SQLite3DB::Statement st=m_db->Prepare("SELECT tblThread.ThreadID FROM tblThread INNER JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE tblThread.BoardID=? AND tblThreadPost.MessageID=?;");\r
st.Bind(0,boardid);\r
st.Bind(1,messageid);\r
\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ std::string temp2("");\r
+ StringFunctions::Convert(boardid,temp1);\r
+ StringFunctions::Convert(messageid,temp2);\r
+\r
+ logmessage+="initial bound boardid=" + temp1 + " messageid=" + temp2 + " | ";\r
+ }\r
+\r
st.Step();\r
if(st.RowReturned())\r
{\r
st.ResultInt(0,threadid);\r
+\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ StringFunctions::Convert(threadid,temp1);\r
+\r
+ logmessage+="result threadid=" + temp1 + " | ";\r
+ }\r
}\r
else\r
{\r
st.Bind(1,(*i).m_messageid);\r
st.Step();\r
\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ std::string temp2("");\r
+ StringFunctions::Convert(boardid,temp1);\r
+ StringFunctions::Convert((*i).m_messageid,temp2);\r
+\r
+ logmessage+="find bound boardid=" + temp1 + " messageid=" + temp2 + " | ";\r
+ }\r
+\r
if(st.RowReturned())\r
{\r
st.ResultInt(0,threadid);\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ StringFunctions::Convert(threadid,temp1);\r
+\r
+ logmessage+="find result threadid=" + temp1 + " | ";\r
+ }\r
+ }\r
+ else\r
+ {\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ StringFunctions::Convert(threadid,temp1);\r
+\r
+ logmessage+="find not found | ";\r
+ }\r
}\r
\r
st.Reset();\r
st.Bind(0,boardid);\r
st.Step(true);\r
threadid=st.GetLastInsertRowID();\r
+\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ std::string temp2("");\r
+ StringFunctions::Convert(boardid,temp1);\r
+ StringFunctions::Convert(threadid,temp2);\r
+\r
+ logmessage+="insert thread bind boardid=" + temp1 + " result threadid=" + temp2 + " | ";\r
+ }\r
+\r
}\r
}\r
\r
st2.Bind(2,threadid);\r
st2.Step();\r
\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ std::string temp2("");\r
+ std::string temp3("");\r
+ StringFunctions::Convert(m_threadmessages[0].m_messageid,temp1);\r
+ StringFunctions::Convert(m_threadmessages[m_threadmessages.size()-1].m_messageid,temp2);\r
+ StringFunctions::Convert(threadid,temp3);\r
+\r
+ logmessage+="update bind fmessageid=" + temp1 + " lmessageid=" + temp2 + " threadid=" + temp3 + " | ";\r
+ }\r
+\r
SQLite3DB::Statement st3=m_db->Prepare("DELETE FROM tblThreadPost WHERE ThreadID=?;");\r
st3.Bind(0,threadid);\r
st3.Step();\r
\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ StringFunctions::Convert(threadid,temp1);\r
+\r
+ logmessage+="delete bind threadid=" + temp1 + " | ";\r
+ }\r
+\r
SQLite3DB::Statement deleteotherst=m_db->Prepare("DELETE FROM tblThread WHERE ThreadID IN (SELECT tblThread.ThreadID FROM tblThreadPost INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID WHERE tblThread.BoardID=? AND tblThreadPost.MessageID=?);");\r
\r
count=0;\r
deleteotherst.Step();\r
deleteotherst.Reset();\r
\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ std::string temp2("");\r
+ StringFunctions::Convert(boardid,temp1);\r
+ StringFunctions::Convert((*i).m_messageid,temp2);\r
+\r
+ logmessage+="deleteother bind boardid=" + temp1 + " messageid=" + temp2 + " | ";\r
+ }\r
+\r
st4.Bind(0,threadid);\r
st4.Bind(1,(*i).m_messageid);\r
st4.Bind(2,count);\r
st4.Step();\r
st4.Reset();\r
+\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ std::string temp2("");\r
+ std::string temp3("");\r
+ StringFunctions::Convert(threadid,temp1);\r
+ StringFunctions::Convert((*i).m_messageid,temp2);\r
+ StringFunctions::Convert(count,temp3);\r
+\r
+ logmessage+="insertthreadpost bind threadid=" + temp1 + " messageid=" + temp2 + " count=" + temp3 + " | ";\r
+ }\r
+\r
}\r
}\r
else\r
st2.Bind(0,threadid);\r
st2.Step();\r
\r
+ if(ll=="8")\r
+ {\r
+ std::string temp1("");\r
+ StringFunctions::Convert(threadid,temp1);\r
+\r
+ logmessage+="delete thread bind threadid=" + temp1 + " | ";\r
+ }\r
+\r
m_log->trace("ThreadBuilder::Build deleted thread");\r
}\r
\r
- m_db->Execute("COMMIT;");\r
+ st.Finalize();\r
+\r
+ //m_db->Execute("COMMIT;");\r
+\r
+ if(ll=="8")\r
+ {\r
+ m_log->trace(logmessage);\r
+ }\r
\r
return true;\r
\r
projects / fms.git / commitdiff