src/dbmaintenancethread.cpp

   1 #include "../include/dbmaintenancethread.h"\r
   2 #include "../include/stringfunctions.h"\r
   3 #include "../include/option.h"\r
   4 #include "../include/threadbuilder.h"\r
   5 #include "../include/dbsetup.h"\r
   6 \r
   7 #include <vector>\r
   8 \r
   9 #include <Poco/Timestamp.h>\r
  10 #include <Poco/Timespan.h>\r
  11 #include <Poco/DateTimeFormatter.h>\r
  12 #include <Poco/Thread.h>\r
  13 \r
  14 DBMaintenanceThread::DBMaintenanceThread()\r
  15 {\r
  16         // move last maintenance times back so they will all run soon\r
  17         m_last10minute=Poco::Timestamp();\r
  18         m_last30minute=Poco::Timestamp();\r
  19         m_last30minute-=Poco::Timespan(0,0,11,0,0);\r
  20         m_last1hour=Poco::Timestamp();\r
  21         m_last1hour-=Poco::Timespan(0,0,49,0,0);\r
  22         m_last6hour=Poco::Timestamp();\r
  23         m_last6hour-=Poco::Timespan(0,5,42,0,0);\r
  24         m_last1day=Poco::Timestamp();\r
  25         m_last1day-=Poco::Timespan(0,23,51,0,0);\r
  26 }\r
  27 \r
  28 \r
  29 void DBMaintenanceThread::Do10MinuteMaintenance()\r
  30 {\r
  31         std::vector<std::pair<long,long> > m_unthreadedmessages;\r
  32         Option option(m_db);\r
  33         std::string ll("");\r
  34         option.Get("LogLevel",ll);\r
  35 \r
  36         ThreadBuilder tb(m_db);\r
  37         SQLite3DB::Statement boardst=m_db->Prepare("SELECT BoardID FROM tblBoard WHERE Forum='true';");\r
  38         // select messages for a board that aren't in a thread\r
  39         // This query was causing the db to be locked and a journal file created.\r
  40         // build a list of boards and messageids and then use that instead of keeping the query in use\r
  41         SQLite3DB::Statement selectst=m_db->Prepare("SELECT tblMessage.MessageID FROM tblMessage \\r
  42                                                                                                 INNER JOIN tblMessageBoard ON tblMessage.MessageID=tblMessageBoard.MessageID \\r
  43                                                                                                 LEFT JOIN tblThreadPost ON tblMessage.MessageID=tblThreadPost.MessageID \\r
  44                                                                                                 WHERE tblMessageBoard.BoardID=? AND tblThreadPost.MessageID IS NULL;");\r
  45 \r
  46         boardst.Step();\r
  47         while(boardst.RowReturned())\r
  48         {\r
  49                 int boardid=-1;\r
  50                 boardst.ResultInt(0,boardid);\r
  51                 boardst.Step();\r
  52 \r
  53                 selectst.Bind(0,boardid);\r
  54                 selectst.Step();\r
  55 \r
  56                 while(selectst.RowReturned())\r
  57                 {\r
  58                         int messageid=-1;\r
  59 \r
  60                         selectst.ResultInt(0,messageid);\r
  61 \r
  62                         m_unthreadedmessages.push_back(std::pair<long,long>(boardid,messageid));\r
  63 \r
  64                         selectst.Step();\r
  65                 }\r
  66                 selectst.Reset();\r
  67         }\r
  68         selectst.Finalize();\r
  69         boardst.Finalize();\r
  70 \r
  71         for(std::vector<std::pair<long,long> >::iterator i=m_unthreadedmessages.begin(); i!=m_unthreadedmessages.end(); i++)\r
  72         {\r
  73                 tb.Build((*i).second,(*i).first,true);\r
  74         }\r
  75         \r
  76         /*\r
  77         while(boardst.RowReturned())\r
  78         {\r
  79                 int boardid=-1;\r
  80 \r
  81                 boardst.ResultInt(0,boardid);\r
  82 \r
  83                 selectst.Bind(0,boardid);\r
  84                 selectst.Step();\r
  85 \r
  86                 while(selectst.RowReturned())\r
  87                 {\r
  88                         int messageid=-1;\r
  89 \r
  90                         selectst.ResultInt(0,messageid);\r
  91 \r
  92                         tb.Build(messageid,boardid,true);\r
  93 \r
  94                         selectst.Step();\r
  95                 }\r
  96                 selectst.Reset();\r
  97 \r
  98                 boardst.Step();\r
  99                 boardst.Reset();\r
 100         }\r
 101         */\r
 102 \r
 103         // now rebuild threads where the message has been deleted\r
 104         SQLite3DB::Statement st=m_db->Prepare("SELECT tblThreadPost.MessageID, tblThread.BoardID FROM tblThreadPost INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID LEFT JOIN tblMessage ON tblThreadPost.MessageID=tblMessage.MessageID WHERE tblMessage.MessageID IS NULL;");\r
 105         st.Step();\r
 106         while(st.RowReturned())\r
 107         {\r
 108                 int messageid=-1;\r
 109                 int boardid=-1;\r
 110 \r
 111                 st.ResultInt(0,messageid);\r
 112                 st.ResultInt(1,boardid);\r
 113 \r
 114                 tb.Build(messageid,boardid,true);\r
 115 \r
 116                 st.Step();\r
 117         }\r
 118 \r
 119         // delete threads that have no messages\r
 120         m_db->Execute("DELETE FROM tblThread WHERE ThreadID IN (SELECT tblThread.ThreadID FROM tblThread LEFT JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE tblThreadPost.ThreadID IS NULL);");\r
 121 \r
 122         // TODO - remove after corruption issue fixed\r
 123         if(ll=="8")\r
 124         {\r
 125                 std::string dbres=TestDBIntegrity(m_db);\r
 126                 m_log->trace("DBMaintenanceThread::Do10MinuteMaintenance() end TestDBIntegrity returned "+dbres);\r
 127                 if(dbres!="ok")\r
 128                 {\r
 129                         m_db->Execute("REINDEX;");\r
 130                         dbres=TestDBIntegrity(m_db);\r
 131                         m_log->trace("DBMaintenanceThread::Do10MinuteMaintenenace() end after reindex returned "+dbres);\r
 132                 }\r
 133         }\r
 134 \r
 135         m_log->debug("PeriodicDBMaintenance::Do10MinuteMaintenance");\r
 136 }\r
 137 \r
 138 void DBMaintenanceThread::Do30MinuteMaintenance()\r
 139 {\r
 140         // UNCOMMENT method in run when code is placed here\r
 141         m_log->debug("PeriodicDBMaintenance::Do30MinuteMaintenance");\r
 142 }\r
 143 \r
 144 void DBMaintenanceThread::Do1HourMaintenance()\r
 145 {\r
 146 \r
 147         m_db->Execute("BEGIN;");\r
 148         // recalculate all trust levels - this is CPU instensive\r
 149         // do 1 identity at a time as doing it with 1 UPDATE statement locks that database for the duration\r
 150         SQLite3DB::Statement st=m_db->Prepare("SELECT TargetIdentityID,PeerMessageTrust,PeerTrustListTrust FROM vwCalculatedPeerTrust;");\r
 151         SQLite3DB::Statement upd=m_db->Prepare("UPDATE tblIdentity SET PeerMessageTrust=?, PeerTrustListTrust=? WHERE IdentityID=?");\r
 152         st.Step();\r
 153         while(st.RowReturned())\r
 154         {\r
 155                 int identityid=0;\r
 156                 int trust=0;\r
 157                 \r
 158                 st.ResultInt(0,identityid);\r
 159 \r
 160                 upd.Bind(0,identityid);\r
 161                 if(st.ResultNull(1)==false)\r
 162                 {\r
 163                         trust=0;\r
 164                         st.ResultInt(1,trust);\r
 165                         upd.Bind(0,trust);\r
 166                 }\r
 167                 else\r
 168                 {\r
 169                         upd.Bind(0);\r
 170                 }\r
 171                 if(st.ResultNull(2)==false)\r
 172                 {\r
 173                         trust=0;\r
 174                         st.ResultInt(2,trust);\r
 175                         upd.Bind(1,trust);\r
 176                 }\r
 177                 else\r
 178                 {\r
 179                         upd.Bind(1);\r
 180                 }\r
 181                 upd.Bind(2,identityid);\r
 182                 upd.Step();\r
 183                 upd.Reset();\r
 184 \r
 185                 st.Step();\r
 186         }\r
 187 \r
 188         // set null peer trust for identities without a calculated trust\r
 189         st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE IdentityID NOT IN (SELECT TargetIdentityID FROM vwCalculatedPeerTrust);");\r
 190         upd=m_db->Prepare("UPDATE tblIdentity SET PeerMessageTrust=NULL, PeerTrustListTrust=NULL WHERE IdentityID=?;");\r
 191         st.Step();\r
 192         while(st.RowReturned())\r
 193         {\r
 194                 int identityid=0;\r
 195                 st.ResultInt(0,identityid);\r
 196                 upd.Bind(0,identityid);\r
 197                 upd.Step();\r
 198                 upd.Reset();\r
 199                 st.Step();\r
 200         }\r
 201 \r
 202         st.Finalize();\r
 203         upd.Finalize();\r
 204 \r
 205         // insert all identities not in trust list already\r
 206         m_db->Execute("INSERT INTO tblIdentityTrust(LocalIdentityID,IdentityID) SELECT LocalIdentityID,IdentityID FROM tblLocalIdentity,tblIdentity WHERE LocalIdentityID || '_' || IdentityID NOT IN (SELECT LocalIdentityID || '_' || IdentityID FROM tblIdentityTrust);");\r
 207 \r
 208         m_db->Execute("COMMIT;");\r
 209 \r
 210         m_log->debug("PeriodicDBMaintenance::Do1HourMaintenance");\r
 211 }\r
 212 \r
 213 void DBMaintenanceThread::Do6HourMaintenance()\r
 214 {\r
 215 \r
 216         m_db->Execute("BEGIN;");\r
 217 \r
 218         // if we remove a board and the reply boardid is still set to it, we need to replace it with a boardid that does exist\r
 219         SQLite3DB::Statement st=m_db->Prepare("SELECT MessageID FROM tblMessage WHERE ReplyBoardID NOT IN (SELECT BoardID FROM tblBoard);");\r
 220         SQLite3DB::Statement st2=m_db->Prepare("SELECT BoardID FROM tblMessageBoard WHERE MessageID=?;");\r
 221         SQLite3DB::Statement upd=m_db->Prepare("UPDATE tblMessage SET ReplyBoardID=? WHERE MessageID=?;");\r
 222         st.Step();\r
 223         while(st.RowReturned())\r
 224         {\r
 225                 // find a valid boardid for the message\r
 226                 int messageid=0;\r
 227                 int boardid=0;\r
 228 \r
 229                 st.ResultInt(0,messageid);\r
 230 \r
 231                 st2.Bind(0,messageid);\r
 232                 st2.Step();\r
 233                 if(st2.RowReturned())\r
 234                 {\r
 235                         st2.ResultInt(0,boardid);\r
 236                         upd.Bind(0,boardid);\r
 237                         upd.Bind(1,messageid);\r
 238                         upd.Step();\r
 239                         upd.Reset();\r
 240                 }\r
 241                 st2.Reset();\r
 242                 \r
 243                 st.Step();\r
 244         }\r
 245 \r
 246         st.Finalize();\r
 247         st2.Finalize();\r
 248         upd.Finalize();\r
 249 \r
 250         m_db->Execute("COMMIT;");\r
 251 \r
 252         m_log->debug("PeriodicDBMaintenance::Do6HourMaintenance");\r
 253 }\r
 254 \r
 255 void DBMaintenanceThread::Do1DayMaintenance()\r
 256 {\r
 257         Poco::DateTime date;\r
 258 \r
 259         m_db->Execute("BEGIN;");\r
 260 \r
 261         // delete all puzzles 2 or more days old\r
 262         date=Poco::Timestamp();\r
 263         date-=Poco::Timespan(2,0,0,0,0);\r
 264         m_db->Execute("DELETE FROM tblIntroductionPuzzleInserts WHERE Day<='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 265         m_db->Execute("DELETE FROM tblIntroductionPuzzleRequests WHERE Day<='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 266 \r
 267         // delete all identities we've never seen and were added more than 20 days ago\r
 268         date=Poco::Timestamp();\r
 269         date-=Poco::Timespan(20,0,0,0,0);\r
 270         m_db->Execute("DELETE FROM tblIdentity WHERE LastSeen IS NULL AND DateAdded<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 271 \r
 272         // delete old identity requests - we don't need them anymore\r
 273         date=Poco::Timestamp();\r
 274         date-=Poco::Timespan(2,0,0,0,0);\r
 275         m_db->Execute("DELETE FROM tblIdentityRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 276 \r
 277         // delete old board list inserts/requests - we don't need them anymore\r
 278         date=Poco::Timestamp();\r
 279         date-=Poco::Timespan(2,0,0,0,0);\r
 280         m_db->Execute("DELETE FROM tblBoardListInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 281         m_db->Execute("DELETE FROM tblBoardListRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 282 \r
 283         // delete old local identity inserts - we don't need them anymore\r
 284         date=Poco::Timestamp();\r
 285         date-=Poco::Timespan(2,0,0,0,0);\r
 286         m_db->Execute("DELETE FROM tblLocalIdentityInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 287 \r
 288         // delete old message list inserts/requests - we don't need them anymore\r
 289         date=Poco::Timestamp();\r
 290         date-=Poco::Timespan(2,0,0,0,0);\r
 291         m_db->Execute("DELETE FROM tblMessageListInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 292         m_db->Execute("DELETE FROM tblMessageListRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 293 \r
 294         // delete old trust list inserts/requests - we don't need them anymore\r
 295         date=Poco::Timestamp();\r
 296         date-=Poco::Timespan(2,0,0,0,0);\r
 297         m_db->Execute("DELETE FROM tblTrustListInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 298         m_db->Execute("DELETE FROM tblTrustListRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
 299 \r
 300         // delete trust lists from identities we aren't trusting anymore\r
 301         m_db->Execute("DELETE FROM tblPeerTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity WHERE (LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust')) AND (PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust')));");\r
 302 \r
 303         // remove identityid from messages where the identity has been deleted\r
 304         m_db->Execute("UPDATE tblMessage SET IdentityID=NULL WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);");\r
 305 \r
 306         // try to re-attach messages from identities that were previously deleted, but have been since re-added\r
 307         // first get the names from messages that have a NULL IdentityID\r
 308         SQLite3DB::Statement st=m_db->Prepare("SELECT FromName FROM tblMessage WHERE IdentityID IS NULL GROUP BY FromName;");\r
 309         SQLite3DB::Statement findst=m_db->Prepare("SELECT IdentityID,PublicKey FROM tblIdentity WHERE Name=?;");\r
 310         st.Step();\r
 311         while(st.RowReturned())\r
 312         {\r
 313                 std::string name="";\r
 314                 std::string namepart="";\r
 315                 std::string publickey="";\r
 316                 int identityid=0;\r
 317                 st.ResultText(0,name);\r
 318 \r
 319                 std::vector<std::string> parts;\r
 320                 StringFunctions::Split(name,"@",parts);\r
 321 \r
 322                 // name can have a @ in it - so reattach all parts except the last which is the key\r
 323                 for(long i=0; i<parts.size()-1; i++)\r
 324                 {\r
 325                         if(i!=0)\r
 326                         {\r
 327                                 namepart+="@";\r
 328                         }\r
 329                         namepart+=parts[i];\r
 330                 }\r
 331 \r
 332                 // find identities with this name\r
 333                 findst.Bind(0,namepart);\r
 334                 findst.Step();\r
 335                 while(findst.RowReturned())\r
 336                 {\r
 337                         publickey="";\r
 338                         identityid=0;\r
 339                         findst.ResultText(1,publickey);\r
 340                         // check if public key matches 2nd part\r
 341                         if(parts.size()>1 && publickey.find(parts[1])==4)\r
 342                         {\r
 343                                 // we have the identity - so update the messages table with the identityid\r
 344                                 findst.ResultInt(0,identityid);\r
 345 \r
 346                                 SQLite3DB::Statement st3=m_db->Prepare("UPDATE tblMessage SET IdentityID=? WHERE FromName=? AND IdentityID IS NULL;");\r
 347                                 st3.Bind(0,identityid);\r
 348                                 st3.Bind(1,name);\r
 349                                 st3.Step();\r
 350                         }\r
 351                         findst.Step();\r
 352                 }\r
 353                 findst.Reset();\r
 354 \r
 355                 st.Step();\r
 356         }\r
 357 \r
 358         // delete single use identities that are older than 7 days\r
 359         date=Poco::Timestamp();\r
 360         date-=Poco::Timespan(7,0,0,0,0);\r
 361         st=m_db->Prepare("DELETE FROM tblIdentity WHERE SingleUse='true' AND DateAdded<?;");\r
 362         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S"));\r
 363         st.Step();\r
 364 \r
 365         // delete local single use identities that are older than 7 days\r
 366         date=Poco::Timestamp();\r
 367         date-=Poco::Timespan(7,0,0,0,0);\r
 368         st=m_db->Prepare("DELETE FROM tblLocalIdentity WHERE SingleUse='true' AND DateCreated<?;");\r
 369         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S"));\r
 370         st.Step();\r
 371 \r
 372         // delete old messages\r
 373         date=Poco::Timestamp();\r
 374         date-=Poco::Timespan(m_deletemessagesolderthan,0,0,0,0);\r
 375         m_log->trace("PeriodicDBMaintenance::Do1DayMaintenance deleting messages prior to "+Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
 376         st=m_db->Prepare("DELETE FROM tblMessage WHERE MessageDate<?;");\r
 377         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
 378         st.Step();\r
 379 \r
 380         // delete old message requests\r
 381         date=Poco::Timestamp();\r
 382         date-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);\r
 383         st=m_db->Prepare("DELETE FROM tblMessageRequests WHERE Day<?;");\r
 384         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
 385         st.Step();\r
 386 \r
 387         // delete tblIdentityTrust for local identities and identities that have been deleted\r
 388         m_db->Execute("DELETE FROM tblIdentityTrust WHERE LocalIdentityID NOT IN (SELECT LocalIdentityID FROM tblLocalIdentity);");\r
 389         m_db->Execute("DELETE FROM tblIdentityTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);");\r
 390 \r
 391 \r
 392         // reduce failure count for each identity\r
 393         m_db->Execute("UPDATE tblIdentity SET FailureCount=0 WHERE FailureCount<(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');");\r
 394         m_db->Execute("UPDATE tblIdentity SET FailureCount=FailureCount-(SELECT OptionValue FROM tblOption WHERE OptionName='FailureCountReduction') WHERE FailureCount>=(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');");\r
 395 \r
 396         st.Finalize();\r
 397         findst.Finalize();\r
 398 \r
 399         m_db->Execute("COMMIT;");\r
 400 \r
 401         m_log->debug("PeriodicDBMaintenance::Do1DayMaintenance");\r
 402 \r
 403 }\r
 404 \r
 405 void DBMaintenanceThread::run()\r
 406 {\r
 407         m_log->debug("DBMaintenanceThread::run thread started.");\r
 408 \r
 409         LoadDatabase();\r
 410         Option option(m_db);\r
 411         std::string tempval("");\r
 412 \r
 413         m_deletemessagesolderthan=180;\r
 414         tempval="180";\r
 415         option.Get("DeleteMessagesOlderThan",tempval);\r
 416         StringFunctions::Convert(tempval,m_deletemessagesolderthan);\r
 417 \r
 418         m_messagedownloadmaxdaysbackward=5;\r
 419         tempval="5";\r
 420         option.Get("MessageDownloadMaxDaysBackward",tempval);\r
 421         StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
 422 \r
 423 \r
 424         Poco::DateTime now;\r
 425         int i=0;\r
 426 \r
 427         do\r
 428         {\r
 429                 now=Poco::Timestamp();\r
 430 \r
 431                 if((m_last10minute+Poco::Timespan(0,0,10,0,0))<=now)\r
 432                 {\r
 433                         Do10MinuteMaintenance();\r
 434                         m_last10minute=Poco::Timestamp();\r
 435                 }\r
 436                 /*\r
 437                 if((m_last30minute+Poco::Timespan(0,0,30,0,0))<=now)\r
 438                 {\r
 439                         Do30MinuteMaintenance();\r
 440                         m_last30minute=Poco::Timestamp();\r
 441                 }\r
 442                 */\r
 443                 if((m_last1hour+Poco::Timespan(0,1,0,0,0))<=now)\r
 444                 {\r
 445                         Do1HourMaintenance();\r
 446                         m_last1hour=Poco::Timestamp();\r
 447                 }\r
 448                 if((m_last6hour+Poco::Timespan(0,6,0,0,0))<=now)\r
 449                 {\r
 450                         Do6HourMaintenance();\r
 451                         m_last6hour=Poco::Timestamp();\r
 452                 }\r
 453                 if((m_last1day+Poco::Timespan(1,0,0,0,0))<=now)\r
 454                 {\r
 455                         Do1DayMaintenance();\r
 456                         m_last1day=Poco::Timestamp();\r
 457                 }\r
 458 \r
 459                 i=0;\r
 460                 while(i++<5 && !IsCancelled())\r
 461                 {\r
 462                         Poco::Thread::sleep(1000);\r
 463                 }\r
 464 \r
 465         }while(!IsCancelled());\r
 466 \r
 467         m_log->debug("DBMaintenanceThread::run thread exiting.");\r
 468 }\r