#include "../../include/freenet/messagelistrequester.h"\r
#include "../../include/freenet/messagelistxml.h"\r
\r
-#include <Poco/DateTime.h>\r
#include <Poco/DateTimeFormatter.h>\r
#include <Poco/DateTimeParser.h>\r
#include <Poco/Timestamp.h>\r
#include <xmem.h>\r
#endif\r
\r
-MessageListRequester::MessageListRequester()\r
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)\r
{\r
Initialize();\r
}\r
\r
-MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester<long>(fcp)\r
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)\r
{\r
Initialize();\r
}\r
}\r
}\r
\r
-void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)\r
+void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards, const bool forceload)\r
{\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
- st.Step();\r
- while(st.RowReturned())\r
+ // only query database when forced, or an 30 minutes have passed since last query\r
+ if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime())\r
{\r
- std::string boardname="";\r
- std::string tempval="";\r
- st.ResultText(0,boardname);\r
- st.ResultText(1,tempval);\r
-\r
- if(tempval=="true")\r
- {\r
- boards[boardname]=true;\r
- }\r
- else\r
+ m_boardscache.clear();\r
+ SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
+ st.Step();\r
+ while(st.RowReturned())\r
{\r
- boards[boardname]=false;\r
- }\r
+ std::string boardname="";\r
+ std::string tempval="";\r
+ st.ResultText(0,boardname);\r
+ st.ResultText(1,tempval);\r
\r
- st.Step();\r
+ if(tempval=="true")\r
+ {\r
+ m_boardscache[boardname]=true;\r
+ }\r
+ else\r
+ {\r
+ m_boardscache[boardname]=false;\r
+ }\r
+\r
+ st.Step();\r
+ }\r
+ m_boardscacheupdate=Poco::DateTime();\r
}\r
+\r
+ boards=m_boardscache;\r
+\r
}\r
\r
-const bool MessageListRequester::HandleAllData(FCPMessage &message)\r
+const bool MessageListRequester::HandleAllData(FCPv2::Message &message)\r
{ \r
SQLite3DB::Statement st;\r
- SQLite3DB::Statement trustst;\r
std::vector<std::string> idparts;\r
long datalength;\r
std::vector<char> data;\r
MessageListXML xml;\r
long identityid;\r
+ long fromidentityid;\r
long index;\r
std::map<std::string,bool> boards; // list of boards and if we will save messages for that board or not\r
+ std::map<std::string,long> identityids; // list of identity public keys and their id in the database\r
bool addmessage=false;\r
std::string boardsstr="";\r
std::string datestr="";\r
StringFunctions::Convert(idparts[1],identityid);\r
StringFunctions::Convert(idparts[2],index);\r
\r
+ fromidentityid=identityid;\r
+\r
// wait for all data to be received from connection\r
- while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()<datalength)\r
- {\r
- m_fcp->Update(1);\r
- }\r
+ m_fcp->WaitForBytes(1000,datalength);\r
\r
// if we got disconnected- return immediately\r
- if(m_fcp->Connected()==false)\r
+ if(m_fcp->IsConnected()==false)\r
{\r
return false;\r
}\r
\r
// receive the file\r
- data.resize(datalength);\r
- if(data.size()>0)\r
- {\r
- m_fcp->ReceiveRaw(&data[0],datalength);\r
- }\r
+ m_fcp->Receive(data,datalength);\r
\r
// parse file into xml and update the database\r
if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
{\r
\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
+ m_db->Execute("BEGIN;");\r
+\r
SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
- SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");\r
+ SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);");\r
+ SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
+\r
for(long i=0; i<xml.MessageCount(); i++)\r
{\r
\r
\r
if(addmessage==true)\r
{\r
- st.Bind(0,identityid);\r
- st.Bind(1,xml.GetDate(i));\r
- st.Bind(2,xml.GetIndex(i));\r
- st.Step();\r
- if(st.RowReturned()==false)\r
- {\r
- mst.Bind(0,identityid);\r
- mst.Bind(1,xml.GetDate(i));\r
- mst.Bind(2,xml.GetIndex(i));\r
- mst.Step();\r
- mst.Reset();\r
- }\r
- st.Reset();\r
+ mst.Bind(0,identityid);\r
+ mst.Bind(1,xml.GetDate(i));\r
+ mst.Bind(2,xml.GetIndex(i));\r
+ mst.Bind(3,identityid);\r
+ mst.Step();\r
+ mst.Reset();\r
+\r
+ // We need to update ID here, in case this index was already inserted from another\r
+ // identity's message list. This doesn't reset try count - maybe we should if the from\r
+ // identity was another identity\r
+ ust.Bind(0,identityid);\r
+ ust.Bind(1,identityid);\r
+ ust.Bind(2,xml.GetDate(i));\r
+ ust.Bind(3,xml.GetIndex(i));\r
+ ust.Step();\r
+ ust.Reset();\r
+\r
+ m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i));\r
+\r
}\r
else\r
{\r
\r
if(addmessage==true)\r
{\r
- spk.Bind(0,xml.GetExternalIdentity(i));\r
- spk.Step();\r
- if(spk.RowReturned())\r
+ int thisidentityid=0;\r
+ if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end())\r
+ {\r
+ thisidentityid=identityids[xml.GetExternalIdentity(i)];\r
+ }\r
+ else\r
+ {\r
+ spk.Bind(0,xml.GetExternalIdentity(i));\r
+ spk.Step();\r
+\r
+ if(spk.RowReturned())\r
+ {\r
+ spk.ResultInt(0,thisidentityid);\r
+ identityids[xml.GetExternalIdentity(i)]=thisidentityid;\r
+ }\r
+\r
+ spk.Reset();\r
+ }\r
+\r
+ if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end())\r
{\r
- int thisidentityid=0;\r
- spk.ResultInt(0,thisidentityid);\r
mst.Bind(0,thisidentityid);\r
mst.Bind(1,xml.GetExternalDate(i));\r
mst.Bind(2,xml.GetExternalIndex(i));\r
+ mst.Bind(3,fromidentityid);\r
mst.Step();\r
mst.Reset();\r
+\r
+ m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i));\r
}\r
- spk.Reset();\r
}\r
else\r
{\r
st.Step();\r
st.Finalize();\r
\r
+ spk.Finalize();\r
+ mst.Finalize();\r
+ ust.Finalize();\r
+\r
+ m_db->Execute("COMMIT;");\r
+\r
m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
}\r
else\r
// remove this identityid from request list\r
RemoveFromRequestList(identityid);\r
\r
+ // keep 2 days of request indexes in the cache\r
+ while(m_requestindexcache.size()>2)\r
+ {\r
+ m_requestindexcache.erase(m_requestindexcache.begin());\r
+ }\r
+\r
return true;\r
\r
}\r
\r
-const bool MessageListRequester::HandleGetFailed(FCPMessage &message)\r
+const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)\r
{\r
SQLite3DB::Statement st;\r
std::vector<std::string> idparts;\r
void MessageListRequester::Initialize()\r
{\r
m_fcpuniquename="ActiveMessageListRequester";\r
- std::string tempval="";\r
-\r
+ std::string tempval("");\r
m_maxrequests=0;\r
- Option::Instance()->GetInt("MaxMessageListRequests",m_maxrequests);\r
+ Option option(m_db);\r
+\r
+ option.GetInt("MaxMessageListRequests",m_maxrequests);\r
\r
// active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2\r
m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);\r
}\r
\r
tempval="";\r
- Option::Instance()->Get("LocalTrustOverridesPeerTrust",tempval);\r
+ option.Get("LocalTrustOverridesPeerTrust",tempval);\r
if(tempval=="true")\r
{\r
m_localtrustoverrides=true;\r
}\r
\r
tempval="";\r
- Option::Instance()->Get("SaveMessagesFromNewBoards",tempval);\r
+ option.Get("SaveMessagesFromNewBoards",tempval);\r
if(tempval=="true")\r
{\r
m_savetonewboards=true;\r
\r
m_messagedownloadmaxdaysbackward=5;\r
tempval="5";\r
- Option::Instance()->Get("MessageDownloadMaxDaysBackward",tempval);\r
+ option.Get("MessageDownloadMaxDaysBackward",tempval);\r
StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
\r
+ m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0);\r
+\r
}\r
\r
void MessageListRequester::PopulateIDList()\r
// select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
if(m_localtrustoverrides==false)\r
{\r
- st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
}\r
else\r
{\r
- st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
}\r
st.Step();\r
\r
}\r
}\r
\r
-void MessageListRequester::StartRedirectRequest(FCPMessage &message)\r
+void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)\r
{\r
std::vector<std::string> parts;\r
std::string indexstr="";\r
std::string identityidstr="";\r
std::string datestr="";\r
- FCPMessage newmessage;\r
+ FCPv2::Message newmessage;\r
\r
// get the new edition #\r
StringFunctions::Split(message["RedirectURI"],"/",parts);\r
newmessage["ReturnType"]="direct";\r
newmessage["MaxSize"]="1000000";\r
\r
- m_fcp->SendMessage(newmessage);\r
+ m_fcp->Send(newmessage);\r
\r
}\r
\r
void MessageListRequester::StartRequest(const long &id)\r
{\r
Poco::DateTime now;\r
- FCPMessage message;\r
+ FCPv2::Message message;\r
std::string publickey;\r
int index=0;\r
std::string indexstr;\r
message["ReturnType"]="direct";\r
message["MaxSize"]="1000000";\r
\r
- m_fcp->SendMessage(message);\r
+ m_fcp->Send(message);\r
\r
m_requesting.push_back(id);\r
}\r