1 #include "../../include/freenet/messagelistinserter.h"
\r
2 #include "../../include/freenet/messagexml.h"
\r
3 #include "../../include/freenet/messagelistxml.h"
\r
5 #include <Poco/DateTime.h>
\r
6 #include <Poco/Timespan.h>
\r
7 #include <Poco/DateTimeFormatter.h>
\r
13 MessageListInserter::MessageListInserter(SQLite3DB::DB *db):IIndexInserter<long>(db)
\r
18 MessageListInserter::MessageListInserter(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexInserter<long>(db,fcp)
\r
23 void MessageListInserter::CheckForNeededInsert()
\r
26 // more than 15 minutes trying to insert - restart
\r
27 if(m_inserting.size()>0 && (m_laststartedinsert+Poco::Timespan(0,0,15,0,0)<=Poco::DateTime()))
\r
29 m_log->error("MessageListInserter::CheckForNeededInsert more than 15 minutes have passed without success/failure. Clearing inserts.");
\r
30 m_lastinsertedxml[m_inserting[0]]="";
\r
31 m_inserting.clear();
\r
34 // only do 1 insert at a time
\r
35 if(m_inserting.size()==0)
\r
39 Poco::DateTime previous;
\r
40 bool startedinsert=false;
\r
42 // reset the last inserted xml doc to nothing if the day has changed
\r
43 if(m_lastchecked.day()!=now.day())
\r
45 m_lastinsertedxml.clear();
\r
48 previous-=Poco::Timespan(m_daysbackward,0,0,0,0);
\r
50 // query for identities that have messages in the past X days and (we haven't inserted lists for in the past 30 minutes OR identity has a record in tmpMessageListInsert)
\r
51 sql="SELECT tblLocalIdentity.LocalIdentityID ";
\r
52 sql+="FROM tblLocalIdentity INNER JOIN tblMessageInserts ON tblLocalIdentity.LocalIdentityID=tblMessageInserts.LocalIdentityID ";
\r
53 sql+="WHERE tblMessageInserts.Day>=? AND ((tblLocalIdentity.LastInsertedMessageList<=? OR tblLocalIdentity.LastInsertedMessageList IS NULL OR tblLocalIdentity.LastInsertedMessageList='') OR tblLocalIdentity.LocalIdentityID IN (SELECT LocalIdentityID FROM tmpMessageListInsert)) AND tblLocalIdentity.PrivateKey IS NOT NULL AND tblLocalIdentity.PrivateKey <> '' ";
\r
54 sql+="GROUP BY tblLocalIdentity.LocalIdentityID ";
\r
55 sql+="ORDER BY tblLocalIdentity.LastInsertedMessageList;";
\r
57 SQLite3DB::Statement st=m_db->Prepare(sql);
\r
58 st.Bind(0,Poco::DateTimeFormatter::format(previous,"%Y-%m-%d"));
\r
59 st.Bind(1,Poco::DateTimeFormatter::format((now-Poco::Timespan(0,0,30,0,0)),"%Y-%m-%d %H:%M:%S"));
\r
62 while(st.RowReturned() && startedinsert==false)
\r
64 int localidentityid;
\r
65 st.ResultInt(0,localidentityid);
\r
66 startedinsert=StartInsert(localidentityid);
\r
73 const bool MessageListInserter::HandlePutFailed(FCPv2::Message &message)
\r
75 std::vector<std::string> idparts;
\r
76 //std::vector<std::string> uriparts;
\r
77 long localidentityid=0;
\r
80 StringFunctions::Split(message["Identifier"],"|",idparts);
\r
81 //StringFunctions::Split(message["URI"],"/",uriparts);
\r
83 StringFunctions::Convert(idparts[1],localidentityid);
\r
84 //StringFunctions::Convert(uriparts[2],index); - URI won't be set for USKs
\r
86 if(message["Fatal"]=="true" || message["Code"]=="9")
\r
88 SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageListInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'false');");
\r
89 st.Bind(0,localidentityid);
\r
90 st.Bind(1,idparts[2]);
\r
95 RemoveFromInsertList(localidentityid);
\r
97 // reset the last inserted xml doc to nothing so we will try to insert this one again
\r
98 m_lastinsertedxml[localidentityid]="";
\r
100 m_log->trace("MessageListInserter::HandlePutFailed insert failed for "+message["Identifier"]);
\r
106 const bool MessageListInserter::HandlePutSuccessful(FCPv2::Message &message)
\r
108 Poco::DateTime now;
\r
109 std::vector<std::string> idparts;
\r
110 std::vector<std::string> uriparts;
\r
111 long localidentityid;
\r
114 StringFunctions::Split(message["Identifier"],"|",idparts);
\r
115 StringFunctions::Split(message["URI"],"/",uriparts);
\r
117 StringFunctions::Convert(idparts[1],localidentityid);
\r
118 StringFunctions::Convert(uriparts[2],index);
\r
120 SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageListInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'true');");
\r
121 st.Bind(0,localidentityid);
\r
122 st.Bind(1,idparts[2]);
\r
126 now=Poco::Timestamp();
\r
127 st=m_db->Prepare("UPDATE tblLocalIdentity SET LastInsertedMessageList=? WHERE LocalIdentityID=?;");
\r
128 st.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S"));
\r
129 st.Bind(1,localidentityid);
\r
132 // delete only a single record from tmpMessageListInsert
\r
133 st=m_db->Prepare("SELECT MessageListInsertID FROM tmpMessageListInsert WHERE LocalIdentityID=?;");
\r
134 st.Bind(0,localidentityid);
\r
136 if(st.RowReturned())
\r
139 st.ResultInt(0,id);
\r
141 st=m_db->Prepare("DELETE FROM tmpMessageListInsert WHERE MessageListInsertID=?;");
\r
146 RemoveFromInsertList(localidentityid);
\r
148 m_log->debug("MessageListInserter::HandlePutSuccessful successfully inserted MessageList.");
\r
153 void MessageListInserter::Initialize()
\r
155 std::string tempval("");
\r
156 m_fcpuniquename="MessageListInserter";
\r
158 Option option(m_db);
\r
160 option.Get("MessageListDaysBackward",tempval);
\r
161 StringFunctions::Convert(tempval,m_daysbackward);
\r
164 const bool MessageListInserter::StartInsert(const long &localidentityid)
\r
166 FCPv2::Message message;
\r
167 Poco::DateTime date;
\r
168 Poco::DateTime now;
\r
169 std::string privatekey;
\r
170 std::string localidentityidstr;
\r
171 MessageListXML mlxml;
\r
172 MessageXML messxml;
\r
173 std::string xmlstr;
\r
174 std::string xmlsizestr;
\r
176 std::string indexstr;
\r
178 date-=Poco::Timespan(m_daysbackward,0,0,0,0);
\r
179 StringFunctions::Convert(localidentityid,localidentityidstr);
\r
181 SQLite3DB::Statement st=m_db->Prepare("SELECT Day, InsertIndex, MessageXML, PrivateKey FROM tblMessageInserts INNER JOIN tblLocalIdentity ON tblMessageInserts.LocalIdentityID=tblLocalIdentity.LocalIdentityID WHERE tblLocalIdentity.LocalIdentityID=? AND Day>=? AND tblMessageInserts.MessageUUID IS NOT NULL;");
\r
182 st.Bind(0,localidentityid);
\r
183 st.Bind(1,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));
\r
186 while(st.RowReturned())
\r
190 std::string xmlstr;
\r
191 std::vector<std::string> boards;
\r
193 st.ResultText(0,day);
\r
194 st.ResultInt(1,index);
\r
195 st.ResultText(2,xmlstr);
\r
196 st.ResultText(3,privatekey);
\r
198 messxml.ParseXML(xmlstr);
\r
200 mlxml.AddMessage(day,index,messxml.GetBoards());
\r
207 st=m_db->Prepare("SELECT MessageDate, MessageIndex, PublicKey, MessageID, InsertDate FROM tblMessage INNER JOIN tblIdentity ON tblMessage.IdentityID=tblIdentity.IdentityID WHERE MessageIndex IS NOT NULL ORDER BY MessageDate DESC, MessageTime DESC LIMIT 175;");
\r
208 SQLite3DB::Statement st2=m_db->Prepare("SELECT BoardName FROM tblBoard INNER JOIN tblMessageBoard ON tblBoard.BoardID=tblMessageBoard.BoardID WHERE tblMessageBoard.MessageID=?;");
\r
211 while(st.RowReturned())
\r
213 std::string day="";
\r
215 std::string publickey="";
\r
216 std::vector<std::string> boardlist;
\r
218 std::string insertdate="";
\r
220 st.ResultText(0,day);
\r
221 st.ResultInt(1,index);
\r
222 st.ResultText(2,publickey);
\r
223 st.ResultInt(3,messageid);
\r
224 st.ResultText(4,insertdate);
\r
226 st2.Bind(0,messageid);
\r
228 while(st2.RowReturned())
\r
230 std::string boardname="";
\r
231 st2.ResultText(0,boardname);
\r
232 StringFunctions::LowerCase(boardname,boardname);
\r
233 boardlist.push_back(boardname);
\r
238 // TODO - remove insertdate empty check sometime after 0.3.32 release and get rid of using day
\r
241 mlxml.AddExternalMessage(publickey,insertdate,index,boardlist);
\r
245 mlxml.AddExternalMessage(publickey,day,index,boardlist);
\r
251 // get last inserted messagelist index for this day
\r
253 st=m_db->Prepare("SELECT MAX(InsertIndex) FROM tblMessageListInserts WHERE LocalIdentityID=? AND Day=?;");
\r
254 st.Bind(0,localidentityid);
\r
255 st.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));
\r
257 if(st.ResultNull(0)==false)
\r
259 st.ResultInt(0,index);
\r
262 StringFunctions::Convert(index,indexstr);
\r
264 xmlstr=mlxml.GetXML();
\r
266 // only insert if the last message this identity inserted is different than this message
\r
267 if(m_lastinsertedxml[localidentityid]!=xmlstr)
\r
269 StringFunctions::Convert(xmlstr.size(),xmlsizestr);
\r
272 message.SetName("ClientPut");
\r
273 message["URI"]=privatekey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|MessageList|"+indexstr+".xml";
\r
274 message["Identifier"]=m_fcpuniquename+"|"+localidentityidstr+"|"+indexstr+"|"+message["URI"];
\r
275 message["UploadFrom"]="direct";
\r
276 message["DataLength"]=xmlsizestr;
\r
277 m_fcp->Send(message);
\r
278 m_fcp->Send(std::vector<char>(xmlstr.begin(),xmlstr.end()));
\r
282 message.SetName("ClientPutComplexDir");
\r
283 message["URI"]="USK"+privatekey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/0/";
\r
284 message["Identifier"]=m_fcpuniquename+"|"+localidentityidstr+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|"+message["URI"];
\r
285 message["DefaultName"]="MessageList.xml";
\r
286 message["Files.0.Name"]="MessageList.xml";
\r
287 message["Files.0.UploadFrom"]="direct";
\r
288 message["Files.0.DataLength"]=xmlsizestr;
\r
289 m_fcp->Send(message);
\r
290 m_fcp->Send(std::vector<char>(xmlstr.begin(),xmlstr.end()));
\r
292 m_inserting.push_back(localidentityid);
\r
293 m_lastinsertedxml[localidentityid]=xmlstr;
\r
295 m_laststartedinsert=Poco::DateTime();
\r
297 m_log->trace("MessageListInserter::StartInsert started insert of "+message["Identifier"]);
\r
304 // xml was the same one that we inserted 30 minutes ago, reset date so we don't continue checking every minute
\r
305 st=m_db->Prepare("UPDATE tblLocalIdentity SET LastInsertedMessageList=? WHERE LocalIdentityID=?;");
\r
306 st.Bind(0,Poco::DateTimeFormatter::format(Poco::DateTime(),"%Y-%m-%d %H:%M:%S"));
\r
307 st.Bind(1,localidentityid);
\r