#include "../../include/freenet/messagelistrequester.h"\r
#include "../../include/freenet/messagelistxml.h"\r
\r
+#include <Poco/DateTimeFormatter.h>\r
+#include <Poco/DateTimeParser.h>\r
+#include <Poco/Timestamp.h>\r
+\r
#ifdef XMEM\r
#include <xmem.h>\r
#endif\r
\r
-MessageListRequester::MessageListRequester()\r
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)\r
{\r
Initialize();\r
}\r
\r
-MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester<long>(fcp)\r
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)\r
{\r
Initialize();\r
}\r
\r
-const bool MessageListRequester::HandleAllData(FCPMessage &message)\r
+const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) const\r
+{\r
+ std::vector<std::string> dateparts;\r
+ int year=0;\r
+ int month=0;\r
+ int day=0;\r
+ Poco::DateTime today;\r
+\r
+ StringFunctions::Split(datestr,"-",dateparts);\r
+ if(dateparts.size()==3)\r
+ {\r
+ StringFunctions::Convert(dateparts[0],year);\r
+ StringFunctions::Convert(dateparts[1],month);\r
+ StringFunctions::Convert(dateparts[2],day);\r
+ if(today.year()>year || (today.year()==year && today.month()>month) || (today.year()==year && today.month()==month && today.day()>=day))\r
+ {\r
+ return true;\r
+ }\r
+ else\r
+ {\r
+ return false;\r
+ }\r
+ }\r
+ else\r
+ {\r
+ return false;\r
+ }\r
+\r
+}\r
+\r
+const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &datestr) const\r
+{\r
+ Poco::DateTime checkdate;\r
+ Poco::DateTime date;\r
+ int tzdiff=0;\r
+ if(Poco::DateTimeParser::tryParse(datestr,date,tzdiff))\r
+ {\r
+ checkdate-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);\r
+ if(checkdate<=date)\r
+ {\r
+ return true;\r
+ }\r
+ else\r
+ {\r
+ return false;\r
+ }\r
+ }\r
+ else\r
+ {\r
+ return false;\r
+ }\r
+}\r
+\r
+void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards, const bool forceload)\r
+{\r
+ // only query database when forced, or an 30 minutes have passed since last query\r
+ if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime())\r
+ {\r
+ m_boardscache.clear();\r
+ SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");\r
+ st.Step();\r
+ while(st.RowReturned())\r
+ {\r
+ std::string boardname="";\r
+ std::string tempval="";\r
+ st.ResultText(0,boardname);\r
+ st.ResultText(1,tempval);\r
+\r
+ if(tempval=="true")\r
+ {\r
+ m_boardscache[boardname]=true;\r
+ }\r
+ else\r
+ {\r
+ m_boardscache[boardname]=false;\r
+ }\r
+\r
+ st.Step();\r
+ }\r
+ m_boardscacheupdate=Poco::DateTime();\r
+ }\r
+\r
+ boards=m_boardscache;\r
+\r
+}\r
+\r
+const bool MessageListRequester::HandleAllData(FCPv2::Message &message)\r
{ \r
- DateTime now;\r
SQLite3DB::Statement st;\r
- SQLite3DB::Statement trustst;\r
std::vector<std::string> idparts;\r
long datalength;\r
std::vector<char> data;\r
MessageListXML xml;\r
long identityid;\r
+ long fromidentityid;\r
long index;\r
+ std::map<std::string,bool> boards; // list of boards and if we will save messages for that board or not\r
+ std::map<std::string,long> identityids; // list of identity public keys and their id in the database\r
+ bool addmessage=false;\r
+ std::string boardsstr="";\r
+ std::string datestr="";\r
+ std::vector<std::string> dateparts;\r
+\r
+ GetBoardList(boards);\r
\r
- now.SetToGMTime();\r
StringFunctions::Split(message["Identifier"],"|",idparts);\r
StringFunctions::Convert(message["DataLength"],datalength);\r
StringFunctions::Convert(idparts[1],identityid);\r
StringFunctions::Convert(idparts[2],index);\r
\r
+ fromidentityid=identityid;\r
+\r
// wait for all data to be received from connection\r
- while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()<datalength)\r
- {\r
- m_fcp->Update(1);\r
- }\r
+ m_fcp->WaitForBytes(1000,datalength);\r
\r
// if we got disconnected- return immediately\r
- if(m_fcp->Connected()==false)\r
+ if(m_fcp->IsConnected()==false)\r
{\r
return false;\r
}\r
\r
// receive the file\r
- data.resize(datalength);\r
- m_fcp->ReceiveRaw(&data[0],datalength);\r
+ m_fcp->Receive(data,datalength);\r
\r
// parse file into xml and update the database\r
- if(xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
+ if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)\r
{\r
\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
- SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");\r
+ m_db->Execute("BEGIN;");\r
+\r
+ SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");\r
+ SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);");\r
+ SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;");\r
+\r
for(long i=0; i<xml.MessageCount(); i++)\r
{\r
- st.Bind(0,identityid);\r
- st.Bind(1,xml.GetDate(i));\r
- st.Bind(2,xml.GetIndex(i));\r
- st.Step();\r
- if(st.RowReturned()==false)\r
+\r
+ // go through each board the message was posted to and see if we are saving messages to that board\r
+ // if the board isn't found, see if we are saving messages to new boards\r
+ boardsstr="";\r
+ addmessage=false;\r
+ std::vector<std::string> messageboards=xml.GetBoards(i);\r
+ for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
+ {\r
+ if(boards.find((*j))!=boards.end())\r
+ {\r
+ if(boards[(*j)]==true)\r
+ {\r
+ addmessage=true;\r
+ }\r
+ }\r
+ else if(m_savetonewboards==true)\r
+ {\r
+ addmessage=true;\r
+ }\r
+ if(j!=messageboards.begin())\r
+ {\r
+ boardsstr+=", ";\r
+ }\r
+ boardsstr+=(*j);\r
+ }\r
+\r
+ if(CheckDateNotFuture(xml.GetDate(i))==false)\r
+ {\r
+ addmessage=false;\r
+ m_log->error(m_fcpuniquename+"::HandleAllData date for message is in future! "+xml.GetDate(i));\r
+ }\r
+\r
+ if(addmessage==true && CheckDateWithinMaxDays(xml.GetDate(i))==false)\r
+ {\r
+ addmessage=false;\r
+ }\r
+\r
+ if(addmessage==true)\r
{\r
mst.Bind(0,identityid);\r
mst.Bind(1,xml.GetDate(i));\r
mst.Bind(2,xml.GetIndex(i));\r
+ mst.Bind(3,identityid);\r
mst.Step();\r
mst.Reset();\r
+\r
+ // We need to update ID here, in case this index was already inserted from another\r
+ // identity's message list. This doesn't reset try count - maybe we should if the from\r
+ // identity was another identity\r
+ ust.Bind(0,identityid);\r
+ ust.Bind(1,identityid);\r
+ ust.Bind(2,xml.GetDate(i));\r
+ ust.Bind(3,xml.GetIndex(i));\r
+ ust.Step();\r
+ ust.Reset();\r
+\r
+ m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i));\r
+\r
+ }\r
+ else\r
+ {\r
+ //m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr+" on "+xml.GetDate(i));\r
+ }\r
+ }\r
+\r
+ // insert external message indexes\r
+ for(long i=0; i<xml.ExternalMessageCount(); i++)\r
+ {\r
+ if(xml.GetExternalType(i)=="Keyed")\r
+ {\r
+ // go through each board the message was posted to and see if we are saving messages to that board\r
+ // if the board isn't found, see if we are saving messages to new boards\r
+ boardsstr="";\r
+ addmessage=false;\r
+ std::vector<std::string> messageboards=xml.GetExternalBoards(i);\r
+ for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)\r
+ {\r
+ if(boards.find((*j))!=boards.end())\r
+ {\r
+ if(boards[(*j)]==true)\r
+ {\r
+ addmessage=true;\r
+ }\r
+ }\r
+ else if(m_savetonewboards==true)\r
+ {\r
+ addmessage=true;\r
+ }\r
+ if(j!=messageboards.begin())\r
+ {\r
+ boardsstr+=", ";\r
+ }\r
+ boardsstr+=(*j);\r
+ }\r
+\r
+ if(CheckDateNotFuture(xml.GetExternalDate(i))==false)\r
+ {\r
+ addmessage=false;\r
+ m_log->error(m_fcpuniquename+"::HandleAllData date for external message is in future! "+xml.GetExternalDate(i));\r
+ }\r
+\r
+ if(addmessage==true && CheckDateWithinMaxDays(xml.GetExternalDate(i))==false)\r
+ {\r
+ addmessage=false;\r
+ }\r
+\r
+ if(addmessage==true)\r
+ {\r
+ int thisidentityid=0;\r
+ if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end())\r
+ {\r
+ thisidentityid=identityids[xml.GetExternalIdentity(i)];\r
+ }\r
+ else\r
+ {\r
+ spk.Bind(0,xml.GetExternalIdentity(i));\r
+ spk.Step();\r
+\r
+ if(spk.RowReturned())\r
+ {\r
+ spk.ResultInt(0,thisidentityid);\r
+ identityids[xml.GetExternalIdentity(i)]=thisidentityid;\r
+ }\r
+\r
+ spk.Reset();\r
+ }\r
+\r
+ if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end())\r
+ {\r
+ mst.Bind(0,thisidentityid);\r
+ mst.Bind(1,xml.GetExternalDate(i));\r
+ mst.Bind(2,xml.GetExternalIndex(i));\r
+ mst.Bind(3,fromidentityid);\r
+ mst.Step();\r
+ mst.Reset();\r
+\r
+ m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i));\r
+ }\r
+ }\r
+ else\r
+ {\r
+ //m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i) + " on " + xml.GetExternalDate(i));\r
+ }\r
}\r
- st.Reset();\r
}\r
- mst.Finalize();\r
- st.Finalize();\r
\r
st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");\r
st.Bind(0,identityid);\r
st.Step();\r
st.Finalize();\r
\r
- m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,"MessageListRequester::HandleAllData parsed TrustList XML file : "+message["Identifier"]);\r
+ spk.Finalize();\r
+ mst.Finalize();\r
+ ust.Finalize();\r
+\r
+ m_db->Execute("COMMIT;");\r
+\r
+ m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);\r
}\r
else\r
{\r
st.Step();\r
st.Finalize();\r
\r
- m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageListRequester::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);\r
+ m_log->error(m_fcpuniquename+"::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);\r
}\r
\r
// remove this identityid from request list\r
RemoveFromRequestList(identityid);\r
\r
+ // keep 2 days of request indexes in the cache\r
+ while(m_requestindexcache.size()>2)\r
+ {\r
+ m_requestindexcache.erase(m_requestindexcache.begin());\r
+ }\r
+\r
return true;\r
\r
}\r
\r
-const bool MessageListRequester::HandleGetFailed(FCPMessage &message)\r
+const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)\r
{\r
- DateTime now;\r
SQLite3DB::Statement st;\r
std::vector<std::string> idparts;\r
long identityid;\r
long index;\r
\r
- now.SetToGMTime();\r
StringFunctions::Split(message["Identifier"],"|",idparts);\r
StringFunctions::Convert(idparts[1],identityid);\r
StringFunctions::Convert(idparts[2],index); \r
\r
+ // code 27 - permanent redirect\r
+ if(message["Code"]=="27")\r
+ {\r
+ StartRedirectRequest(message);\r
+ return true;\r
+ }\r
+\r
// if this is a fatal error - insert index into database so we won't try to download this index again\r
if(message["Fatal"]=="true")\r
{\r
st.Step();\r
st.Finalize();\r
\r
- m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageListRequester::HandleGetFailed fatal error requesting "+message["Identifier"]);\r
+ m_log->error(m_fcpuniquename+"::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);\r
}\r
\r
// remove this identityid from request list\r
\r
void MessageListRequester::Initialize()\r
{\r
- m_fcpuniquename="MessageListRequester";\r
- std::string tempval;\r
- Option::instance()->Get("MaxMessageListRequests",tempval);\r
- StringFunctions::Convert(tempval,m_maxrequests);\r
+ m_fcpuniquename="ActiveMessageListRequester";\r
+ std::string tempval("");\r
+ m_maxrequests=0;\r
+ Option option(m_db);\r
+\r
+ option.GetInt("MaxMessageListRequests",m_maxrequests);\r
+\r
+ // active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2\r
+ m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);\r
+\r
if(m_maxrequests<1)\r
{\r
m_maxrequests=1;\r
- m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"Option MaxMessageListRequests is currently set at "+tempval+". It must be 1 or greater.");\r
+ m_log->error("Option MaxMessageListRequests is currently set at "+tempval+". It must be 1 or greater.");\r
}\r
if(m_maxrequests>100)\r
{\r
- m_log->WriteLog(LogFile::LOGLEVEL_WARNING,"Option MaxMessageListRequests is currently set at "+tempval+". This value might be incorrectly configured.");\r
+ m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+". This value might be incorrectly configured.");\r
+ }\r
+\r
+ tempval="";\r
+ option.Get("LocalTrustOverridesPeerTrust",tempval);\r
+ if(tempval=="true")\r
+ {\r
+ m_localtrustoverrides=true;\r
+ }\r
+ else\r
+ {\r
+ m_localtrustoverrides=false;\r
+ }\r
+\r
+ tempval="";\r
+ option.Get("SaveMessagesFromNewBoards",tempval);\r
+ if(tempval=="true")\r
+ {\r
+ m_savetonewboards=true;\r
+ }\r
+ else\r
+ {\r
+ m_savetonewboards=false;\r
}\r
+\r
+ m_messagedownloadmaxdaysbackward=5;\r
+ tempval="5";\r
+ option.Get("MessageDownloadMaxDaysBackward",tempval);\r
+ StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
+\r
+ m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0);\r
+\r
}\r
\r
void MessageListRequester::PopulateIDList()\r
{\r
- DateTime date;\r
+ Poco::DateTime date;\r
+ Poco::DateTime yesterday=date-Poco::Timespan(1,0,0,0,0);\r
int id;\r
\r
- date.SetToGMTime();\r
+ SQLite3DB::Statement st;\r
\r
// select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)\r
- SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+date.Format("%Y-%m-%d")+"' AND LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ if(m_localtrustoverrides==false)\r
+ {\r
+ st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ }\r
+ else\r
+ {\r
+ st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");\r
+ }\r
st.Step();\r
\r
m_ids.clear();\r
}\r
}\r
\r
+void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)\r
+{\r
+ std::vector<std::string> parts;\r
+ std::string indexstr="";\r
+ std::string identityidstr="";\r
+ std::string datestr="";\r
+ FCPv2::Message newmessage;\r
+\r
+ // get the new edition #\r
+ StringFunctions::Split(message["RedirectURI"],"/",parts);\r
+ //edition # is 2nd to last part\r
+ if(parts.size()>2)\r
+ {\r
+ indexstr=parts[parts.size()-2];\r
+ }\r
+\r
+ // get identityid\r
+ parts.clear();\r
+ StringFunctions::Split(message["Identifier"],"|",parts);\r
+ if(parts.size()>1)\r
+ {\r
+ identityidstr=parts[1];\r
+ }\r
+ if(parts.size()>4)\r
+ {\r
+ datestr=parts[4];\r
+ }\r
+\r
+ newmessage.SetName("ClientGet");\r
+ newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);\r
+ newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];\r
+ newmessage["ReturnType"]="direct";\r
+ newmessage["MaxSize"]="1000000";\r
+\r
+ m_fcp->Send(newmessage);\r
+\r
+}\r
+\r
void MessageListRequester::StartRequest(const long &id)\r
{\r
- DateTime now;\r
- FCPMessage message;\r
+ Poco::DateTime now;\r
+ FCPv2::Message message;\r
std::string publickey;\r
- int index;\r
+ int index=0;\r
std::string indexstr;\r
std::string identityidstr;\r
\r
{\r
st.ResultText(0,publickey);\r
\r
- now.SetToGMTime();\r
+ now=Poco::Timestamp();\r
\r
SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");\r
- st2.Bind(0,now.Format("%Y-%m-%d"));\r
+ st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));\r
st2.Bind(1,id);\r
st2.Step();\r
\r
if(st2.ResultNull(0)==false)\r
{\r
st2.ResultInt(0,index);\r
+ // don't increment index here - the node will let us know if there is a new edition\r
+ // 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index\r
index++;\r
}\r
}\r
StringFunctions::Convert(id,identityidstr);\r
\r
message.SetName("ClientGet");\r
- message["URI"]=publickey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|MessageList|"+indexstr+".xml";\r
- message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|"+message["URI"];\r
+ message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";\r
+ message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];\r
message["ReturnType"]="direct";\r
- message["MaxSize"]="1000000"; // 1 MB\r
+ message["MaxSize"]="1000000";\r
\r
- m_fcp->SendMessage(message);\r
+ m_fcp->Send(message);\r
\r
m_requesting.push_back(id);\r
}\r