X-Git-Url: https://git.pterodactylus.net/?a=blobdiff_plain;f=src%2Ffreenet%2Fmessagelistrequester.cpp;h=c5cb291a082494b50dd02d71912a2979759ae476;hb=59a5414ec47a2932a7802fcd1d98c4d80166564f;hp=c5ef11c586d9c27d21e89fcc6dc33866d199cacc;hpb=c7fcb4c4bc5012a584add81a9509fc1f84c3c688;p=fms.git

diff --git a/src/freenet/messagelistrequester.cpp b/src/freenet/messagelistrequester.cpp
index c5ef11c..c5cb291 100644
--- a/src/freenet/messagelistrequester.cpp
+++ b/src/freenet/messagelistrequester.cpp
@@ -1,23 +1,104 @@
 #include "../../include/freenet/messagelistrequester.h"
 #include "../../include/freenet/messagelistxml.h"
 
+#include <Poco/DateTime.h>
+#include <Poco/DateTimeFormatter.h>
+#include <Poco/DateTimeParser.h>
+#include <Poco/Timestamp.h>
+
 #ifdef XMEM
 	#include <xmem.h>
 #endif
 
-MessageListRequester::MessageListRequester()
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)
 {
 	Initialize();
 }
 
-MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester<long>(fcp)
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)
 {
 	Initialize();
 }
 
-const bool MessageListRequester::HandleAllData(FCPMessage &message)
+const bool MessageListRequester::CheckDateNotFuture(const std::string &datestr) const
+{
+	std::vector<std::string> dateparts;
+	int year=0;
+	int month=0;
+	int day=0;
+	Poco::DateTime today;
+
+	StringFunctions::Split(datestr,"-",dateparts);
+	if(dateparts.size()==3)
+	{
+		StringFunctions::Convert(dateparts[0],year);
+		StringFunctions::Convert(dateparts[1],month);
+		StringFunctions::Convert(dateparts[2],day);
+		if(today.year()>year || (today.year()==year && today.month()>month) || (today.year()==year && today.month()==month && today.day()>=day))
+		{
+			return true;
+		}
+		else
+		{
+			return false;
+		}
+	}
+	else
+	{
+		return false;
+	}
+
+}
+
+const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &datestr) const
+{
+	Poco::DateTime checkdate;
+	Poco::DateTime date;
+	int tzdiff=0;
+	if(Poco::DateTimeParser::tryParse(datestr,date,tzdiff))
+	{
+		checkdate-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);
+		if(checkdate<=date)
+		{
+			return true;
+		}
+		else
+		{
+			return false;
+		}
+	}
+	else
+	{
+		return false;
+	}
+}
+
+void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)
+{
+	SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");
+	st.Step();
+	while(st.RowReturned())
+	{
+		std::string boardname="";
+		std::string tempval="";
+		st.ResultText(0,boardname);
+		st.ResultText(1,tempval);
+
+		if(tempval=="true")
+		{
+			boards[boardname]=true;
+		}
+		else
+		{
+			boards[boardname]=false;
+		}
+
+		st.Step();
+	}
+}
+
+const bool MessageListRequester::HandleAllData(FCPv2::Message &message)
 {	
-	DateTime now;
 	SQLite3DB::Statement st;
 	SQLite3DB::Statement trustst;
 	std::vector<std::string> idparts;
@@ -26,53 +107,164 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 	MessageListXML xml;
 	long identityid;
 	long index;
+	std::map<std::string,bool> boards;	// list of boards and if we will save messages for that board or not
+	bool addmessage=false;
+	std::string boardsstr="";
+	std::string datestr="";
+	std::vector<std::string> dateparts;
+
+	GetBoardList(boards);
 
-	now.SetToGMTime();
 	StringFunctions::Split(message["Identifier"],"|",idparts);
 	StringFunctions::Convert(message["DataLength"],datalength);
 	StringFunctions::Convert(idparts[1],identityid);
 	StringFunctions::Convert(idparts[2],index);
 
 	// wait for all data to be received from connection
-	while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()<datalength)
-	{
-		m_fcp->Update(1);
-	}
+	m_fcp->WaitForBytes(1000,datalength);
 
 	// if we got disconnected- return immediately
-	if(m_fcp->Connected()==false)
+	if(m_fcp->IsConnected()==false)
 	{
 		return false;
 	}
 
 	// receive the file
-	data.resize(datalength);
-	m_fcp->ReceiveRaw(&data[0],datalength);
+	m_fcp->Receive(data,datalength);
 
 	// parse file into xml and update the database
-	if(xml.ParseXML(std::string(data.begin(),data.end()))==true)
+	if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)
 	{
 
+		m_db->Execute("BEGIN;");
+
 		SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");
+		SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");
 		SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");
 		for(long i=0; i<xml.MessageCount(); i++)
 		{
-			st.Bind(0,identityid);
-			st.Bind(1,xml.GetDate(i));
-			st.Bind(2,xml.GetIndex(i));
-			st.Step();
-			if(st.RowReturned()==false)
+
+			// go through each board the message was posted to and see if we are saving messages to that board
+			// if the board isn't found, see if we are saving messages to new boards
+			boardsstr="";
+			addmessage=false;
+			std::vector<std::string> messageboards=xml.GetBoards(i);
+			for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)
+			{
+				if(boards.find((*j))!=boards.end())
+				{
+					if(boards[(*j)]==true)
+					{
+						addmessage=true;
+					}
+				}
+				else if(m_savetonewboards==true)
+				{
+					addmessage=true;
+				}
+				if(j!=messageboards.begin())
+				{
+					boardsstr+=", ";
+				}
+				boardsstr+=(*j);
+			}
+
+			if(CheckDateNotFuture(xml.GetDate(i))==false)
+			{
+				addmessage=false;
+				m_log->error(m_fcpuniquename+"::HandleAllData date for message is in future! "+xml.GetDate(i));
+			}
+
+			if(addmessage==true && CheckDateWithinMaxDays(xml.GetDate(i))==false)
+			{
+				addmessage=false;
+			}
+
+			if(addmessage==true)
+			{
+				st.Bind(0,identityid);
+				st.Bind(1,xml.GetDate(i));
+				st.Bind(2,xml.GetIndex(i));
+				st.Step();
+				if(st.RowReturned()==false)
+				{
+					mst.Bind(0,identityid);
+					mst.Bind(1,xml.GetDate(i));
+					mst.Bind(2,xml.GetIndex(i));
+					mst.Step();
+					mst.Reset();
+				}
+				st.Reset();
+			}
+			else
+			{
+				//m_log->trace("MessageListRequester::HandleAllData will not download message posted to "+boardsstr+" on "+xml.GetDate(i));
+			}
+		}
+
+		// insert external message indexes
+		for(long i=0; i<xml.ExternalMessageCount(); i++)
+		{
+			if(xml.GetExternalType(i)=="Keyed")
 			{
-				mst.Bind(0,identityid);
-				mst.Bind(1,xml.GetDate(i));
-				mst.Bind(2,xml.GetIndex(i));
-				mst.Step();
-				mst.Reset();
+				// go through each board the message was posted to and see if we are saving messages to that board
+				// if the board isn't found, see if we are saving messages to new boards
+				boardsstr="";
+				addmessage=false;
+				std::vector<std::string> messageboards=xml.GetExternalBoards(i);
+				for(std::vector<std::string>::iterator j=messageboards.begin(); j!=messageboards.end(); j++)
+				{
+					if(boards.find((*j))!=boards.end())
+					{
+						if(boards[(*j)]==true)
+						{
+							addmessage=true;
+						}
+					}
+					else if(m_savetonewboards==true)
+					{
+						addmessage=true;
+					}
+					if(j!=messageboards.begin())
+					{
+						boardsstr+=", ";
+					}
+					boardsstr+=(*j);
+				}
+
+				if(CheckDateNotFuture(xml.GetExternalDate(i))==false)
+				{
+					addmessage=false;
+					m_log->error(m_fcpuniquename+"::HandleAllData date for external message is in future! "+xml.GetExternalDate(i));
+				}
+
+				if(addmessage==true && CheckDateWithinMaxDays(xml.GetExternalDate(i))==false)
+				{
+					addmessage=false;
+				}
+
+				if(addmessage==true)
+				{
+					spk.Bind(0,xml.GetExternalIdentity(i));
+					spk.Step();
+					if(spk.RowReturned())
+					{
+						int thisidentityid=0;
+						spk.ResultInt(0,thisidentityid);
+						mst.Bind(0,thisidentityid);
+						mst.Bind(1,xml.GetExternalDate(i));
+						mst.Bind(2,xml.GetExternalIndex(i));
+						mst.Step();
+						mst.Reset();
+					}
+					spk.Reset();
+				}
+				else
+				{
+					//m_log->trace("MessageListRequester::HandleAllData will not download external message posted to "+boardsstr+" from " + xml.GetExternalIdentity(i) + " on " + xml.GetExternalDate(i));
+				}
 			}
-			st.Reset();
 		}
-		mst.Finalize();
-		st.Finalize();
 
 		st=m_db->Prepare("INSERT INTO tblMessageListRequests(IdentityID,Day,RequestIndex,Found) VALUES(?,?,?,'true');");
 		st.Bind(0,identityid);
@@ -81,7 +273,9 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 		st.Step();
 		st.Finalize();
 
-		m_log->WriteLog(LogFile::LOGLEVEL_DEBUG,"MessageListRequester::HandleAllData parsed MessageList XML file : "+message["Identifier"]);
+		m_db->Execute("COMMIT;");
+
+		m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);
 	}
 	else
 	{
@@ -93,7 +287,7 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 		st.Step();
 		st.Finalize();
 
-		m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageListRequester::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);
+		m_log->error(m_fcpuniquename+"::HandleAllData error parsing MessageList XML file : "+message["Identifier"]);
 	}
 
 	// remove this identityid from request list
@@ -103,19 +297,24 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 
 }
 
-const bool MessageListRequester::HandleGetFailed(FCPMessage &message)
+const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)
 {
-	DateTime now;
 	SQLite3DB::Statement st;
 	std::vector<std::string> idparts;
 	long identityid;
 	long index;
 
-	now.SetToGMTime();
 	StringFunctions::Split(message["Identifier"],"|",idparts);
 	StringFunctions::Convert(idparts[1],identityid);
 	StringFunctions::Convert(idparts[2],index);	
 
+	// code 27 - permanent redirect
+	if(message["Code"]=="27")
+	{
+		StartRedirectRequest(message);
+		return true;
+	}
+
 	// if this is a fatal error - insert index into database so we won't try to download this index again
 	if(message["Fatal"]=="true")
 	{
@@ -126,7 +325,7 @@ const bool MessageListRequester::HandleGetFailed(FCPMessage &message)
 		st.Step();
 		st.Finalize();
 
-		m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"MessageListRequester::HandleGetFailed fatal error requesting "+message["Identifier"]);
+		m_log->error(m_fcpuniquename+"::HandleGetFailed fatal error code="+message["Code"]+" requesting "+message["Identifier"]);
 	}
 
 	// remove this identityid from request list
@@ -137,30 +336,72 @@ const bool MessageListRequester::HandleGetFailed(FCPMessage &message)
 
 void MessageListRequester::Initialize()
 {
-	m_fcpuniquename="MessageListRequester";
-	std::string tempval;
-	Option::Instance()->Get("MaxMessageListRequests",tempval);
-	StringFunctions::Convert(tempval,m_maxrequests);
+	m_fcpuniquename="ActiveMessageListRequester";
+	std::string tempval("");
+	m_maxrequests=0;
+	Option option(m_db);
+
+	option.GetInt("MaxMessageListRequests",m_maxrequests);
+
+	// active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2
+	m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);
+
 	if(m_maxrequests<1)
 	{
 		m_maxrequests=1;
-		m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"Option MaxMessageListRequests is currently set at "+tempval+".  It must be 1 or greater.");
+		m_log->error("Option MaxMessageListRequests is currently set at "+tempval+".  It must be 1 or greater.");
 	}
 	if(m_maxrequests>100)
 	{
-		m_log->WriteLog(LogFile::LOGLEVEL_WARNING,"Option MaxMessageListRequests is currently set at "+tempval+".  This value might be incorrectly configured.");
+		m_log->warning("Option MaxMessageListRequests is currently set at "+tempval+".  This value might be incorrectly configured.");
+	}
+
+	tempval="";
+	option.Get("LocalTrustOverridesPeerTrust",tempval);
+	if(tempval=="true")
+	{
+		m_localtrustoverrides=true;
+	}
+	else
+	{
+		m_localtrustoverrides=false;
+	}
+
+	tempval="";
+	option.Get("SaveMessagesFromNewBoards",tempval);
+	if(tempval=="true")
+	{
+		m_savetonewboards=true;
 	}
+	else
+	{
+		m_savetonewboards=false;
+	}
+
+	m_messagedownloadmaxdaysbackward=5;
+	tempval="5";
+	option.Get("MessageDownloadMaxDaysBackward",tempval);
+	StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);
+
 }
 
 void MessageListRequester::PopulateIDList()
 {
-	DateTime date;
+	Poco::DateTime date;
+	Poco::DateTime yesterday=date-Poco::Timespan(1,0,0,0,0);
 	int id;
 
-	date.SetToGMTime();
+	SQLite3DB::Statement st;
 
 	// select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)
-	SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+date.Format("%Y-%m-%d")+"' AND LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
+	if(m_localtrustoverrides==false)
+	{
+		st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
+	}
+	else
+	{
+		st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
+	}
 	st.Step();
 
 	m_ids.clear();
@@ -173,12 +414,50 @@ void MessageListRequester::PopulateIDList()
 	}
 }
 
+void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)
+{
+	std::vector<std::string> parts;
+	std::string indexstr="";
+	std::string identityidstr="";
+	std::string datestr="";
+	FCPv2::Message newmessage;
+
+	// get the new edition #
+	StringFunctions::Split(message["RedirectURI"],"/",parts);
+	//edition # is 2nd to last part
+	if(parts.size()>2)
+	{
+		indexstr=parts[parts.size()-2];
+	}
+
+	// get identityid
+	parts.clear();
+	StringFunctions::Split(message["Identifier"],"|",parts);
+	if(parts.size()>1)
+	{
+		identityidstr=parts[1];
+	}
+	if(parts.size()>4)
+	{
+		datestr=parts[4];
+	}
+
+	newmessage.SetName("ClientGet");
+	newmessage["URI"]=StringFunctions::UriDecode(message["RedirectURI"]);
+	newmessage["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+datestr+"|"+newmessage["URI"];
+	newmessage["ReturnType"]="direct";
+	newmessage["MaxSize"]="1000000";
+
+	m_fcp->Send(newmessage);
+
+}
+
 void MessageListRequester::StartRequest(const long &id)
 {
-	DateTime now;
-	FCPMessage message;
+	Poco::DateTime now;
+	FCPv2::Message message;
 	std::string publickey;
-	int index;
+	int index=0;
 	std::string indexstr;
 	std::string identityidstr;
 
@@ -190,10 +469,10 @@ void MessageListRequester::StartRequest(const long &id)
 	{
 		st.ResultText(0,publickey);
 
-		now.SetToGMTime();
+		now=Poco::Timestamp();
 
 		SQLite3DB::Statement st2=m_db->Prepare("SELECT MAX(RequestIndex) FROM tblMessageListRequests WHERE Day=? AND IdentityID=?;");
-		st2.Bind(0,now.Format("%Y-%m-%d"));
+		st2.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));
 		st2.Bind(1,id);
 		st2.Step();
 
@@ -203,6 +482,8 @@ void MessageListRequester::StartRequest(const long &id)
 			if(st2.ResultNull(0)==false)
 			{
 				st2.ResultInt(0,index);
+				// don't increment index here - the node will let us know if there is a new edition
+				// 2008-05-31 - well actually the node isn't reliably retreiving the latest edition for USKs, so we DO need to increment the index
 				index++;
 			}
 		}
@@ -212,12 +493,12 @@ void MessageListRequester::StartRequest(const long &id)
 		StringFunctions::Convert(id,identityidstr);
 
 		message.SetName("ClientGet");
-		message["URI"]=publickey+m_messagebase+"|"+now.Format("%Y-%m-%d")+"|MessageList|"+indexstr+".xml";
-		message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|"+message["URI"];
+		message["URI"]="USK"+publickey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/"+indexstr+"/MessageList.xml";
+		message["Identifier"]=m_fcpuniquename+"|"+identityidstr+"|"+indexstr+"|_|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];
 		message["ReturnType"]="direct";
-		message["MaxSize"]="1000000";			// 1 MB
+		message["MaxSize"]="1000000";
 
-		m_fcp->SendMessage(message);
+		m_fcp->Send(message);
 
 		m_requesting.push_back(id);
 	}