src/freenet/messagelistinserter.cpp

   1 #include "../../include/freenet/messagelistinserter.h"\r
   2 #include "../../include/freenet/messagexml.h"\r
   3 #include "../../include/freenet/messagelistxml.h"\r
   4 \r
   5 #include <Poco/DateTime.h>\r
   6 #include <Poco/Timespan.h>\r
   7 #include <Poco/DateTimeFormatter.h>\r
   8 \r
   9 #ifdef XMEM\r
  10         #include <xmem.h>\r
  11 #endif\r
  12 \r
  13 MessageListInserter::MessageListInserter(SQLite3DB::DB *db):IIndexInserter<long>(db)\r
  14 {\r
  15         Initialize();\r
  16 }\r
  17 \r
  18 MessageListInserter::MessageListInserter(SQLite3DB::DB *db, FCPv2::Connection *fcp):IIndexInserter<long>(db,fcp)\r
  19 {\r
  20         Initialize();\r
  21 }\r
  22 \r
  23 void MessageListInserter::CheckForNeededInsert()\r
  24 {\r
  25 \r
  26         // more than 10 minutes trying to insert - restart\r
  27         if(m_inserting.size()>0 && (m_laststartedinsert+Poco::Timespan(0,0,10,0,0)<=Poco::DateTime()))\r
  28         {\r
  29                 m_log->error("MessageListInserter::CheckForNeededInsert more than 10 minutes have passed without success/failure.  Clearing inserts.");\r
  30                 m_inserting.clear();\r
  31         }\r
  32 \r
  33         // only do 1 insert at a time\r
  34         if(m_inserting.size()==0)\r
  35         {\r
  36                 std::string sql;\r
  37                 Poco::DateTime now;\r
  38                 Poco::DateTime previous;\r
  39                 bool startedinsert=false;\r
  40 \r
  41                 // reset the last inserted xml doc to nothing if the day has changed\r
  42                 if(m_lastchecked.day()!=now.day())\r
  43                 {\r
  44                         m_lastinsertedxml.clear();\r
  45                 }\r
  46 \r
  47                 previous-=Poco::Timespan(m_daysbackward,0,0,0,0);\r
  48 \r
  49                 // query for identities that have messages in the past X days and (we haven't inserted lists for in the past 30 minutes OR identity has a record in tmpMessageListInsert)\r
  50                 sql="SELECT tblLocalIdentity.LocalIdentityID ";\r
  51                 sql+="FROM tblLocalIdentity INNER JOIN tblMessageInserts ON tblLocalIdentity.LocalIdentityID=tblMessageInserts.LocalIdentityID ";\r
  52                 sql+="WHERE tblMessageInserts.Day>=? AND ((tblLocalIdentity.LastInsertedMessageList<=? OR tblLocalIdentity.LastInsertedMessageList IS NULL OR tblLocalIdentity.LastInsertedMessageList='') OR tblLocalIdentity.LocalIdentityID IN (SELECT LocalIdentityID FROM tmpMessageListInsert)) ";\r
  53                 sql+="GROUP BY tblLocalIdentity.LocalIdentityID ";\r
  54                 sql+="ORDER BY tblLocalIdentity.LastInsertedMessageList;";\r
  55 \r
  56                 SQLite3DB::Statement st=m_db->Prepare(sql);\r
  57                 st.Bind(0,Poco::DateTimeFormatter::format(previous,"%Y-%m-%d"));\r
  58                 st.Bind(1,Poco::DateTimeFormatter::format((now-Poco::Timespan(0,0,30,0,0)),"%Y-%m-%d %H:%M:%S"));\r
  59                 st.Step();\r
  60 \r
  61                 while(st.RowReturned() && startedinsert==false)\r
  62                 {\r
  63                         int localidentityid;\r
  64                         st.ResultInt(0,localidentityid);\r
  65                         startedinsert=StartInsert(localidentityid);\r
  66                         st.Step();\r
  67                 }\r
  68         }\r
  69 \r
  70 }\r
  71 \r
  72 const bool MessageListInserter::HandlePutFailed(FCPv2::Message &message)\r
  73 {\r
  74         std::vector<std::string> idparts;\r
  75         long localidentityid;\r
  76         long index;\r
  77 \r
  78         StringFunctions::Split(message["Identifier"],"|",idparts);\r
  79 \r
  80         // non USK\r
  81         if(idparts[0]==m_fcpuniquename)\r
  82         {\r
  83                 StringFunctions::Convert(idparts[1],localidentityid);\r
  84                 StringFunctions::Convert(idparts[2],index);\r
  85 \r
  86                 if(message["Fatal"]=="true" || message["Code"]=="9")\r
  87                 {\r
  88                         SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageListInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'false');");\r
  89                         st.Bind(0,localidentityid);\r
  90                         st.Bind(1,idparts[4]);\r
  91                         st.Bind(2,index);\r
  92                         st.Step();\r
  93                 }\r
  94 \r
  95                 RemoveFromInsertList(localidentityid);\r
  96 \r
  97                 // reset the last inserted xml doc to nothing so we will try to insert this one again\r
  98                 m_lastinsertedxml[localidentityid]="";\r
  99         }\r
 100         else\r
 101         {\r
 102                 m_log->debug("MessageListInserter::HandlePutFailed "+message["Identifier"]);\r
 103         }\r
 104 \r
 105         return true;\r
 106 \r
 107 }\r
 108 \r
 109 const bool MessageListInserter::HandlePutSuccessful(FCPv2::Message &message)\r
 110 {\r
 111         Poco::DateTime now;\r
 112         std::vector<std::string> idparts;\r
 113         long localidentityid;\r
 114         long index;\r
 115 \r
 116         StringFunctions::Split(message["Identifier"],"|",idparts);\r
 117 \r
 118         // non USK\r
 119         if(idparts[0]==m_fcpuniquename)\r
 120         {\r
 121                 StringFunctions::Convert(idparts[1],localidentityid);\r
 122                 StringFunctions::Convert(idparts[2],index);\r
 123 \r
 124                 SQLite3DB::Statement st=m_db->Prepare("INSERT INTO tblMessageListInserts(LocalIdentityID,Day,InsertIndex,Inserted) VALUES(?,?,?,'true');");\r
 125                 st.Bind(0,localidentityid);\r
 126                 st.Bind(1,idparts[4]);\r
 127                 st.Bind(2,index);\r
 128                 st.Step();\r
 129 \r
 130                 now=Poco::Timestamp();\r
 131                 st=m_db->Prepare("UPDATE tblLocalIdentity SET LastInsertedMessageList=? WHERE LocalIdentityID=?;");\r
 132                 st.Bind(0,Poco::DateTimeFormatter::format(now,"%Y-%m-%d %H:%M:%S"));\r
 133                 st.Bind(1,localidentityid);\r
 134                 st.Step();\r
 135 \r
 136                 // delete only a single record from tmpMessageListInsert\r
 137                 st=m_db->Prepare("SELECT MessageListInsertID FROM tmpMessageListInsert WHERE LocalIdentityID=?;");\r
 138                 st.Bind(0,localidentityid);\r
 139                 st.Step();\r
 140                 if(st.RowReturned())\r
 141                 {\r
 142                         int id=-1;\r
 143                         st.ResultInt(0,id);\r
 144 \r
 145                         st=m_db->Prepare("DELETE FROM tmpMessageListInsert WHERE MessageListInsertID=?;");\r
 146                         st.Bind(0,id);\r
 147                         st.Step();\r
 148                 }\r
 149 \r
 150                 RemoveFromInsertList(localidentityid);\r
 151 \r
 152                 m_log->debug("MessageListInserter::HandlePutSuccessful successfully inserted MessageList.");\r
 153         }\r
 154         else\r
 155         {\r
 156                 m_log->debug("MessageListInserter::HandlePutSuccessful inserted USK MessageList "+message["Identifier"]);\r
 157         }\r
 158 \r
 159         return true;\r
 160 }\r
 161 \r
 162 void MessageListInserter::Initialize()\r
 163 {\r
 164         std::string tempval("");\r
 165         m_fcpuniquename="MessageListInserter";\r
 166         m_daysbackward=0;\r
 167         Option option(m_db);\r
 168 \r
 169         option.Get("MessageListDaysBackward",tempval);\r
 170         StringFunctions::Convert(tempval,m_daysbackward);\r
 171 }\r
 172 \r
 173 const bool MessageListInserter::StartInsert(const long &localidentityid)\r
 174 {\r
 175         FCPv2::Message message;\r
 176         Poco::DateTime date;\r
 177         Poco::DateTime now;\r
 178         std::string privatekey;\r
 179         std::string localidentityidstr;\r
 180         MessageListXML mlxml;\r
 181         MessageXML messxml;\r
 182         std::string xmlstr;\r
 183         std::string xmlsizestr;\r
 184         int index;\r
 185         std::string indexstr;\r
 186 \r
 187         date-=Poco::Timespan(m_daysbackward,0,0,0,0);\r
 188         StringFunctions::Convert(localidentityid,localidentityidstr);\r
 189 \r
 190         SQLite3DB::Statement st=m_db->Prepare("SELECT Day, InsertIndex, MessageXML, PrivateKey FROM tblMessageInserts INNER JOIN tblLocalIdentity ON tblMessageInserts.LocalIdentityID=tblLocalIdentity.LocalIdentityID WHERE tblLocalIdentity.LocalIdentityID=? AND Day>=? AND tblMessageInserts.MessageUUID IS NOT NULL;");\r
 191         st.Bind(0,localidentityid);\r
 192         st.Bind(1,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
 193         st.Step();\r
 194 \r
 195         while(st.RowReturned())\r
 196         {\r
 197                 std::string day;\r
 198                 int index;\r
 199                 std::string xmlstr;\r
 200                 std::vector<std::string> boards;\r
 201 \r
 202                 st.ResultText(0,day);\r
 203                 st.ResultInt(1,index);\r
 204                 st.ResultText(2,xmlstr);\r
 205                 st.ResultText(3,privatekey);\r
 206 \r
 207                 messxml.ParseXML(xmlstr);\r
 208 \r
 209                 mlxml.AddMessage(day,index,messxml.GetBoards());\r
 210 \r
 211                 st.Step();\r
 212         }\r
 213         st.Finalize();\r
 214 \r
 215 \r
 216         st=m_db->Prepare("SELECT MessageDate, MessageIndex, PublicKey, MessageID, InsertDate FROM tblMessage INNER JOIN tblIdentity ON tblMessage.IdentityID=tblIdentity.IdentityID WHERE MessageIndex IS NOT NULL ORDER BY MessageDate DESC, MessageTime DESC LIMIT 175;");\r
 217         SQLite3DB::Statement st2=m_db->Prepare("SELECT BoardName FROM tblBoard INNER JOIN tblMessageBoard ON tblBoard.BoardID=tblMessageBoard.BoardID WHERE tblMessageBoard.MessageID=?;");\r
 218         st.Step();\r
 219 \r
 220         while(st.RowReturned())\r
 221         {\r
 222                 std::string day="";\r
 223                 int index=0;\r
 224                 std::string publickey="";\r
 225                 std::vector<std::string> boardlist;\r
 226                 int messageid=0;\r
 227                 std::string insertdate="";\r
 228                 \r
 229                 st.ResultText(0,day);\r
 230                 st.ResultInt(1,index);\r
 231                 st.ResultText(2,publickey);\r
 232                 st.ResultInt(3,messageid);\r
 233                 st.ResultText(4,insertdate);\r
 234 \r
 235                 st2.Bind(0,messageid);\r
 236                 st2.Step();\r
 237                 while(st2.RowReturned())\r
 238                 {\r
 239                         std::string boardname="";\r
 240                         st2.ResultText(0,boardname);\r
 241                         StringFunctions::LowerCase(boardname,boardname);\r
 242                         boardlist.push_back(boardname);\r
 243                         st2.Step();\r
 244                 }\r
 245                 st2.Reset();\r
 246 \r
 247                 // TODO - remove insertdate empty check sometime after 0.3.32 release and get rid of using day\r
 248                 if(insertdate!="")\r
 249                 {\r
 250                         mlxml.AddExternalMessage(publickey,insertdate,index,boardlist);\r
 251                 }\r
 252                 else\r
 253                 {\r
 254                         mlxml.AddExternalMessage(publickey,day,index,boardlist);\r
 255                 }\r
 256 \r
 257                 st.Step();\r
 258         }\r
 259 \r
 260         // get last inserted messagelist index for this day\r
 261         index=0;\r
 262         st=m_db->Prepare("SELECT MAX(InsertIndex) FROM tblMessageListInserts WHERE LocalIdentityID=? AND Day=?;");\r
 263         st.Bind(0,localidentityid);\r
 264         st.Bind(1,Poco::DateTimeFormatter::format(now,"%Y-%m-%d"));\r
 265         st.Step();\r
 266         if(st.ResultNull(0)==false)\r
 267         {\r
 268                 st.ResultInt(0,index);\r
 269                 index++;\r
 270         }\r
 271         StringFunctions::Convert(index,indexstr);\r
 272 \r
 273         xmlstr=mlxml.GetXML();\r
 274 \r
 275         // only insert if the last message this identity inserted is different than this message\r
 276         if(m_lastinsertedxml[localidentityid]!=xmlstr)\r
 277         {\r
 278                 std::string targeturi="";\r
 279                 StringFunctions::Convert(xmlstr.size(),xmlsizestr);\r
 280 \r
 281                 message.SetName("ClientPut");\r
 282                 message["URI"]=privatekey+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y-%m-%d")+"|MessageList|"+indexstr+".xml";\r
 283                 message["Identifier"]=m_fcpuniquename+"|"+localidentityidstr+"|"+indexstr+"|"+message["URI"];\r
 284                 message["UploadFrom"]="direct";\r
 285                 message["DataLength"]=xmlsizestr;\r
 286                 m_fcp->Send(message);\r
 287                 m_fcp->Send(std::vector<char>(xmlstr.begin(),xmlstr.end()));\r
 288 \r
 289                 message.Clear();\r
 290                 message.SetName("ClientPutComplexDir");\r
 291                 message["URI"]="USK"+privatekey.substr(3)+m_messagebase+"|"+Poco::DateTimeFormatter::format(now,"%Y.%m.%d")+"|MessageList/0/";\r
 292                 message["Identifier"]=m_fcpuniquename+"USK|"+message["URI"];\r
 293                 message["DefaultName"]="MessageList.xml";\r
 294                 message["Files.0.Name"]="MessageList.xml";\r
 295                 message["Files.0.UploadFrom"]="direct";\r
 296                 message["Files.0.DataLength"]=xmlsizestr;\r
 297                 m_fcp->Send(message);\r
 298                 m_fcp->Send(std::vector<char>(xmlstr.begin(),xmlstr.end()));\r
 299 \r
 300                 m_inserting.push_back(localidentityid);\r
 301                 m_lastinsertedxml[localidentityid]=xmlstr;\r
 302 \r
 303                 m_laststartedinsert=Poco::DateTime();\r
 304 \r
 305                 return true;\r
 306         }\r
 307         else\r
 308         {\r
 309 \r
 310                 // xml was the same one that we inserted 30 minutes ago, reset date so we don't continue checking every minute\r
 311                 st=m_db->Prepare("UPDATE tblLocalIdentity SET LastInsertedMessageList=? WHERE LocalIdentityID=?;");\r
 312                 st.Bind(0,Poco::DateTimeFormatter::format(Poco::DateTime(),"%Y-%m-%d %H:%M:%S"));\r
 313                 st.Bind(1,localidentityid);\r
 314                 st.Step();\r
 315 \r
 316                 return false;\r
 317         }\r
 318 \r
 319 }\r