X-Git-Url: https://git.pterodactylus.net/?a=blobdiff_plain;f=src%2Ffreenet%2Fmessagelistrequester.cpp;h=e54e0f232c46f891b1c9c6e1828d07a2c2a8a163;hb=ed0732b2550c23c05fc9faf925620e87ee6dee12;hp=d8ed8aa25ae10b51a8e55cfbcd6cc53b2d2d6aa7;hpb=e662ea47fba8715474851ceebacba400984ee433;p=fms.git

diff --git a/src/freenet/messagelistrequester.cpp b/src/freenet/messagelistrequester.cpp
index d8ed8aa..e54e0f2 100644
--- a/src/freenet/messagelistrequester.cpp
+++ b/src/freenet/messagelistrequester.cpp
@@ -1,7 +1,6 @@
 #include "../../include/freenet/messagelistrequester.h"
 #include "../../include/freenet/messagelistxml.h"
 
-#include <Poco/DateTime.h>
 #include <Poco/DateTimeFormatter.h>
 #include <Poco/DateTimeParser.h>
 #include <Poco/Timestamp.h>
@@ -10,12 +9,12 @@
 	#include <xmem.h>
 #endif
 
-MessageListRequester::MessageListRequester()
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db):IIndexRequester<long>(db)
 {
 	Initialize();
 }
 
-MessageListRequester::MessageListRequester(FCPv2 *fcp):IIndexRequester<long>(fcp)
+MessageListRequester::MessageListRequester(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexRequester<long>(db,fcp)
 {
 	Initialize();
 }
@@ -73,41 +72,51 @@ const bool MessageListRequester::CheckDateWithinMaxDays(const std::string &dates
 	}
 }
 
-void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards)
+void MessageListRequester::GetBoardList(std::map<std::string,bool> &boards, const bool forceload)
 {
-	SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");
-	st.Step();
-	while(st.RowReturned())
+	// only query database when forced, or an 30 minutes have passed since last query
+	if(forceload==true || m_boardscacheupdate+Poco::Timespan(0,0,30,0,0)<=Poco::DateTime())
 	{
-		std::string boardname="";
-		std::string tempval="";
-		st.ResultText(0,boardname);
-		st.ResultText(1,tempval);
-
-		if(tempval=="true")
-		{
-			boards[boardname]=true;
-		}
-		else
+		m_boardscache.clear();
+		SQLite3DB::Statement st=m_db->Prepare("SELECT BoardName, SaveReceivedMessages FROM tblBoard;");
+		st.Step();
+		while(st.RowReturned())
 		{
-			boards[boardname]=false;
-		}
+			std::string boardname="";
+			std::string tempval="";
+			st.ResultText(0,boardname);
+			st.ResultText(1,tempval);
 
-		st.Step();
+			if(tempval=="true")
+			{
+				m_boardscache[boardname]=true;
+			}
+			else
+			{
+				m_boardscache[boardname]=false;
+			}
+
+			st.Step();
+		}
+		m_boardscacheupdate=Poco::DateTime();
 	}
+
+	boards=m_boardscache;
+
 }
 
-const bool MessageListRequester::HandleAllData(FCPMessage &message)
+const bool MessageListRequester::HandleAllData(FCPv2::Message &message)
 {	
 	SQLite3DB::Statement st;
-	SQLite3DB::Statement trustst;
 	std::vector<std::string> idparts;
 	long datalength;
 	std::vector<char> data;
 	MessageListXML xml;
 	long identityid;
+	long fromidentityid;
 	long index;
 	std::map<std::string,bool> boards;	// list of boards and if we will save messages for that board or not
+	std::map<std::string,long> identityids;	// list of identity public keys and their id in the database
 	bool addmessage=false;
 	std::string boardsstr="";
 	std::string datestr="";
@@ -120,32 +129,30 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 	StringFunctions::Convert(idparts[1],identityid);
 	StringFunctions::Convert(idparts[2],index);
 
+	fromidentityid=identityid;
+
 	// wait for all data to be received from connection
-	while(m_fcp->Connected() && m_fcp->ReceiveBufferSize()<datalength)
-	{
-		m_fcp->Update(1);
-	}
+	m_fcp->WaitForBytes(1000,datalength);
 
 	// if we got disconnected- return immediately
-	if(m_fcp->Connected()==false)
+	if(m_fcp->IsConnected()==false)
 	{
 		return false;
 	}
 
 	// receive the file
-	data.resize(datalength);
-	if(data.size()>0)
-	{
-		m_fcp->ReceiveRaw(&data[0],datalength);
-	}
+	m_fcp->Receive(data,datalength);
 
 	// parse file into xml and update the database
 	if(data.size()>0 && xml.ParseXML(std::string(data.begin(),data.end()))==true)
 	{
 
-		SQLite3DB::Statement st=m_db->Prepare("SELECT IdentityID FROM tblMessageRequests WHERE IdentityID=? AND Day=? AND RequestIndex=?;");
+		m_db->Execute("BEGIN;");
+
 		SQLite3DB::Statement spk=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE PublicKey=?;");
-		SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList) VALUES(?,?,?,'true');");
+		SQLite3DB::Statement mst=m_db->Prepare("INSERT INTO tblMessageRequests(IdentityID,Day,RequestIndex,FromMessageList,FromIdentityID) VALUES(?,?,?,'true',?);");
+		SQLite3DB::Statement ust=m_db->Prepare("UPDATE tblMessageRequests SET FromIdentityID=? WHERE IdentityID=? AND Day=? AND RequestIndex=?;");
+
 		for(long i=0; i<xml.MessageCount(); i++)
 		{
 
@@ -187,19 +194,25 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 
 			if(addmessage==true)
 			{
-				st.Bind(0,identityid);
-				st.Bind(1,xml.GetDate(i));
-				st.Bind(2,xml.GetIndex(i));
-				st.Step();
-				if(st.RowReturned()==false)
-				{
-					mst.Bind(0,identityid);
-					mst.Bind(1,xml.GetDate(i));
-					mst.Bind(2,xml.GetIndex(i));
-					mst.Step();
-					mst.Reset();
-				}
-				st.Reset();
+				mst.Bind(0,identityid);
+				mst.Bind(1,xml.GetDate(i));
+				mst.Bind(2,xml.GetIndex(i));
+				mst.Bind(3,identityid);
+				mst.Step();
+				mst.Reset();
+
+				// We need to update ID here, in case this index was already inserted from another
+				// identity's message list.  This doesn't reset try count - maybe we should if the from
+				// identity was another identity
+				ust.Bind(0,identityid);
+				ust.Bind(1,identityid);
+				ust.Bind(2,xml.GetDate(i));
+				ust.Bind(3,xml.GetIndex(i));
+				ust.Step();
+				ust.Reset();
+
+				m_requestindexcache[xml.GetDate(i)][identityid].insert(xml.GetIndex(i));
+
 			}
 			else
 			{
@@ -250,19 +263,36 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 
 				if(addmessage==true)
 				{
-					spk.Bind(0,xml.GetExternalIdentity(i));
-					spk.Step();
-					if(spk.RowReturned())
+					int thisidentityid=0;
+					if(identityids.find(xml.GetExternalIdentity(i))!=identityids.end())
+					{
+						thisidentityid=identityids[xml.GetExternalIdentity(i)];
+					}
+					else
+					{
+						spk.Bind(0,xml.GetExternalIdentity(i));
+						spk.Step();
+
+						if(spk.RowReturned())
+						{
+							spk.ResultInt(0,thisidentityid);
+							identityids[xml.GetExternalIdentity(i)]=thisidentityid;
+						}
+
+						spk.Reset();
+					}
+
+					if(thisidentityid!=0 && m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].find(xml.GetExternalIndex(i))==m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].end())
 					{
-						int thisidentityid=0;
-						spk.ResultInt(0,thisidentityid);
 						mst.Bind(0,thisidentityid);
 						mst.Bind(1,xml.GetExternalDate(i));
 						mst.Bind(2,xml.GetExternalIndex(i));
+						mst.Bind(3,fromidentityid);
 						mst.Step();
 						mst.Reset();
+
+						m_requestindexcache[xml.GetExternalDate(i)][thisidentityid].insert(xml.GetExternalIndex(i));
 					}
-					spk.Reset();
 				}
 				else
 				{
@@ -278,6 +308,12 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 		st.Step();
 		st.Finalize();
 
+		spk.Finalize();
+		mst.Finalize();
+		ust.Finalize();
+
+		m_db->Execute("COMMIT;");
+
 		m_log->debug(m_fcpuniquename+"::HandleAllData parsed MessageList XML file : "+message["Identifier"]);
 	}
 	else
@@ -296,11 +332,17 @@ const bool MessageListRequester::HandleAllData(FCPMessage &message)
 	// remove this identityid from request list
 	RemoveFromRequestList(identityid);
 
+	// keep 2 days of request indexes in the cache
+	while(m_requestindexcache.size()>2)
+	{
+		m_requestindexcache.erase(m_requestindexcache.begin());
+	}
+
 	return true;
 
 }
 
-const bool MessageListRequester::HandleGetFailed(FCPMessage &message)
+const bool MessageListRequester::HandleGetFailed(FCPv2::Message &message)
 {
 	SQLite3DB::Statement st;
 	std::vector<std::string> idparts;
@@ -340,10 +382,11 @@ const bool MessageListRequester::HandleGetFailed(FCPMessage &message)
 void MessageListRequester::Initialize()
 {
 	m_fcpuniquename="ActiveMessageListRequester";
-	std::string tempval="";
-
+	std::string tempval("");
 	m_maxrequests=0;
-	Option::Instance()->GetInt("MaxMessageListRequests",m_maxrequests);
+	Option option(m_db);
+
+	option.GetInt("MaxMessageListRequests",m_maxrequests);
 
 	// active identities get 1/2 of the max requests option + any remaining if not evenly divisible - inactive identities get 1/2
 	m_maxrequests=(m_maxrequests/2)+(m_maxrequests%2);
@@ -359,7 +402,7 @@ void MessageListRequester::Initialize()
 	}
 
 	tempval="";
-	Option::Instance()->Get("LocalTrustOverridesPeerTrust",tempval);
+	option.Get("LocalTrustOverridesPeerTrust",tempval);
 	if(tempval=="true")
 	{
 		m_localtrustoverrides=true;
@@ -370,7 +413,7 @@ void MessageListRequester::Initialize()
 	}
 
 	tempval="";
-	Option::Instance()->Get("SaveMessagesFromNewBoards",tempval);
+	option.Get("SaveMessagesFromNewBoards",tempval);
 	if(tempval=="true")
 	{
 		m_savetonewboards=true;
@@ -382,9 +425,11 @@ void MessageListRequester::Initialize()
 
 	m_messagedownloadmaxdaysbackward=5;
 	tempval="5";
-	Option::Instance()->Get("MessageDownloadMaxDaysBackward",tempval);
+	option.Get("MessageDownloadMaxDaysBackward",tempval);
 	StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);
 
+	m_boardscacheupdate=Poco::DateTime()-Poco::Timespan(1,0,0,0,0);
+
 }
 
 void MessageListRequester::PopulateIDList()
@@ -398,11 +443,11 @@ void MessageListRequester::PopulateIDList()
 	// select identities we want to query (we've seen them today) - sort by their trust level (descending) with secondary sort on how long ago we saw them (ascending)
 	if(m_localtrustoverrides==false)
 	{
-		st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
+		st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust IS NULL OR LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust')) AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
 	}
 	else
 	{
-		st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
+		st=m_db->Prepare("SELECT tblIdentity.IdentityID FROM tblIdentity INNER JOIN vwIdentityStats ON tblIdentity.IdentityID=vwIdentityStats.IdentityID WHERE PublicKey IS NOT NULL AND PublicKey <> '' AND LastSeen>='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"' AND (vwIdentityStats.LastMessageDate>='"+Poco::DateTimeFormatter::format(yesterday,"%Y-%m-%d")+"') AND (LocalMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalMessageTrust') OR (LocalMessageTrust IS NULL AND (PeerMessageTrust IS NULL OR PeerMessageTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerMessageTrust')))) AND FailureCount<=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') ORDER BY LocalMessageTrust+LocalTrustListTrust DESC, LastSeen;");
 	}
 	st.Step();
 
@@ -416,13 +461,13 @@ void MessageListRequester::PopulateIDList()
 	}
 }
 
-void MessageListRequester::StartRedirectRequest(FCPMessage &message)
+void MessageListRequester::StartRedirectRequest(FCPv2::Message &message)
 {
 	std::vector<std::string> parts;
 	std::string indexstr="";
 	std::string identityidstr="";
 	std::string datestr="";
-	FCPMessage newmessage;
+	FCPv2::Message newmessage;
 
 	// get the new edition #
 	StringFunctions::Split(message["RedirectURI"],"/",parts);
@@ -450,14 +495,14 @@ void MessageListRequester::StartRedirectRequest(FCPMessage &message)
 	newmessage["ReturnType"]="direct";
 	newmessage["MaxSize"]="1000000";
 
-	m_fcp->SendMessage(newmessage);
+	m_fcp->Send(newmessage);
 
 }
 
 void MessageListRequester::StartRequest(const long &id)
 {
 	Poco::DateTime now;
-	FCPMessage message;
+	FCPv2::Message message;
 	std::string publickey;
 	int index=0;
 	std::string indexstr;
@@ -500,7 +545,7 @@ void MessageListRequester::StartRequest(const long &id)
 		message["ReturnType"]="direct";
 		message["MaxSize"]="1000000";
 
-		m_fcp->SendMessage(message);
+		m_fcp->Send(message);
 
 		m_requesting.push_back(id);
 	}