version 0.3.33
[fms.git] / src / dbmaintenancethread.cpp
1 #include "../include/dbmaintenancethread.h"\r
2 #include "../include/stringfunctions.h"\r
3 #include "../include/option.h"\r
4 #include "../include/threadbuilder.h"\r
5 #include "../include/dbsetup.h"\r
6 \r
7 #include <vector>\r
8 \r
9 #include <Poco/Timestamp.h>\r
10 #include <Poco/Timespan.h>\r
11 #include <Poco/DateTimeFormatter.h>\r
12 #include <Poco/Thread.h>\r
13 \r
14 DBMaintenanceThread::DBMaintenanceThread()\r
15 {\r
16         // move last maintenance times back so they will all run soon\r
17         m_last10minute=Poco::Timestamp();\r
18         m_last30minute=Poco::Timestamp();\r
19         m_last30minute-=Poco::Timespan(0,0,11,0,0);\r
20         m_last1hour=Poco::Timestamp();\r
21         m_last1hour-=Poco::Timespan(0,0,49,0,0);\r
22         m_last6hour=Poco::Timestamp();\r
23         m_last6hour-=Poco::Timespan(0,5,42,0,0);\r
24         m_last1day=Poco::Timestamp();\r
25         m_last1day-=Poco::Timespan(0,23,51,0,0);\r
26 }\r
27 \r
28 \r
29 void DBMaintenanceThread::Do10MinuteMaintenance()\r
30 {\r
31         std::vector<std::pair<long,long> > m_unthreadedmessages;\r
32         Option option(m_db);\r
33         std::string ll("");\r
34         option.Get("LogLevel",ll);\r
35 \r
36         ThreadBuilder tb(m_db);\r
37         SQLite3DB::Statement boardst=m_db->Prepare("SELECT BoardID FROM tblBoard WHERE Forum='true';");\r
38         // select messages for a board that aren't in a thread\r
39         // This query was causing the db to be locked and a journal file created.\r
40         // build a list of boards and messageids and then use that instead of keeping the query in use\r
41         SQLite3DB::Statement selectst=m_db->Prepare("SELECT tblMessage.MessageID FROM tblMessage \\r
42                                                                                                 INNER JOIN tblMessageBoard ON tblMessage.MessageID=tblMessageBoard.MessageID \\r
43                                                                                                 LEFT JOIN (SELECT tblThread.BoardID, tblThreadPost.MessageID FROM tblThread INNER JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE tblThread.BoardID=?) AS temp1 ON tblMessage.MessageID=temp1.MessageID \\r
44                                                                                                 WHERE tblMessageBoard.BoardID=? AND temp1.BoardID IS NULL;");\r
45 \r
46         boardst.Step();\r
47         while(boardst.RowReturned())\r
48         {\r
49                 int boardid=-1;\r
50                 boardst.ResultInt(0,boardid);\r
51                 boardst.Step();\r
52 \r
53                 selectst.Bind(0,boardid);\r
54                 selectst.Bind(1,boardid);\r
55                 selectst.Step();\r
56 \r
57                 while(selectst.RowReturned())\r
58                 {\r
59                         int messageid=-1;\r
60 \r
61                         selectst.ResultInt(0,messageid);\r
62 \r
63                         m_unthreadedmessages.push_back(std::pair<long,long>(boardid,messageid));\r
64 \r
65                         selectst.Step();\r
66                 }\r
67                 selectst.Reset();\r
68         }\r
69         selectst.Finalize();\r
70         boardst.Finalize();\r
71 \r
72         for(std::vector<std::pair<long,long> >::iterator i=m_unthreadedmessages.begin(); i!=m_unthreadedmessages.end(); i++)\r
73         {\r
74                 tb.Build((*i).second,(*i).first,true);\r
75         }\r
76         \r
77         /*\r
78         while(boardst.RowReturned())\r
79         {\r
80                 int boardid=-1;\r
81 \r
82                 boardst.ResultInt(0,boardid);\r
83 \r
84                 selectst.Bind(0,boardid);\r
85                 selectst.Step();\r
86 \r
87                 while(selectst.RowReturned())\r
88                 {\r
89                         int messageid=-1;\r
90 \r
91                         selectst.ResultInt(0,messageid);\r
92 \r
93                         tb.Build(messageid,boardid,true);\r
94 \r
95                         selectst.Step();\r
96                 }\r
97                 selectst.Reset();\r
98 \r
99                 boardst.Step();\r
100                 boardst.Reset();\r
101         }\r
102         */\r
103 \r
104         // now rebuild threads where the message has been deleted\r
105         SQLite3DB::Statement st=m_db->Prepare("SELECT tblThreadPost.MessageID, tblThread.BoardID FROM tblThreadPost INNER JOIN tblThread ON tblThreadPost.ThreadID=tblThread.ThreadID LEFT JOIN tblMessage ON tblThreadPost.MessageID=tblMessage.MessageID WHERE tblMessage.MessageID IS NULL;");\r
106         st.Step();\r
107         while(st.RowReturned())\r
108         {\r
109                 int messageid=-1;\r
110                 int boardid=-1;\r
111 \r
112                 st.ResultInt(0,messageid);\r
113                 st.ResultInt(1,boardid);\r
114 \r
115                 tb.Build(messageid,boardid,true);\r
116 \r
117                 st.Step();\r
118         }\r
119 \r
120         // delete threads that have no messages\r
121         m_db->Execute("DELETE FROM tblThread WHERE ThreadID IN (SELECT tblThread.ThreadID FROM tblThread LEFT JOIN tblThreadPost ON tblThread.ThreadID=tblThreadPost.ThreadID WHERE tblThreadPost.ThreadID IS NULL);");\r
122 \r
123         // TODO - remove after corruption issue fixed\r
124         if(ll=="8")\r
125         {\r
126                 std::string dbres=TestDBIntegrity(m_db);\r
127                 m_log->trace("DBMaintenanceThread::Do10MinuteMaintenance() end TestDBIntegrity returned "+dbres);\r
128                 if(dbres!="ok")\r
129                 {\r
130                         m_db->Execute("REINDEX;");\r
131                         dbres=TestDBIntegrity(m_db);\r
132                         m_log->trace("DBMaintenanceThread::Do10MinuteMaintenenace() end after reindex returned "+dbres);\r
133                 }\r
134         }\r
135 \r
136         m_log->debug("PeriodicDBMaintenance::Do10MinuteMaintenance");\r
137 }\r
138 \r
139 void DBMaintenanceThread::Do30MinuteMaintenance()\r
140 {\r
141         // UNCOMMENT method in run when code is placed here\r
142         m_log->debug("PeriodicDBMaintenance::Do30MinuteMaintenance");\r
143 }\r
144 \r
145 void DBMaintenanceThread::Do1HourMaintenance()\r
146 {\r
147 \r
148         m_db->Execute("BEGIN;");\r
149         // recalculate all trust levels - this is CPU instensive\r
150         // do 1 identity at a time as doing it with 1 UPDATE statement locks that database for the duration\r
151         SQLite3DB::Statement st=m_db->Prepare("SELECT TargetIdentityID,PeerMessageTrust,PeerTrustListTrust FROM vwCalculatedPeerTrust;");\r
152         SQLite3DB::Statement upd=m_db->Prepare("UPDATE tblIdentity SET PeerMessageTrust=?, PeerTrustListTrust=? WHERE IdentityID=?");\r
153         st.Step();\r
154         while(st.RowReturned())\r
155         {\r
156                 int identityid=0;\r
157                 int trust=0;\r
158                 \r
159                 st.ResultInt(0,identityid);\r
160 \r
161                 upd.Bind(0,identityid);\r
162                 if(st.ResultNull(1)==false)\r
163                 {\r
164                         trust=0;\r
165                         st.ResultInt(1,trust);\r
166                         upd.Bind(0,trust);\r
167                 }\r
168                 else\r
169                 {\r
170                         upd.Bind(0);\r
171                 }\r
172                 if(st.ResultNull(2)==false)\r
173                 {\r
174                         trust=0;\r
175                         st.ResultInt(2,trust);\r
176                         upd.Bind(1,trust);\r
177                 }\r
178                 else\r
179                 {\r
180                         upd.Bind(1);\r
181                 }\r
182                 upd.Bind(2,identityid);\r
183                 upd.Step();\r
184                 upd.Reset();\r
185 \r
186                 st.Step();\r
187         }\r
188 \r
189         // set null peer trust for identities without a calculated trust\r
190         st=m_db->Prepare("SELECT IdentityID FROM tblIdentity WHERE IdentityID NOT IN (SELECT TargetIdentityID FROM vwCalculatedPeerTrust);");\r
191         upd=m_db->Prepare("UPDATE tblIdentity SET PeerMessageTrust=NULL, PeerTrustListTrust=NULL WHERE IdentityID=?;");\r
192         st.Step();\r
193         while(st.RowReturned())\r
194         {\r
195                 int identityid=0;\r
196                 st.ResultInt(0,identityid);\r
197                 upd.Bind(0,identityid);\r
198                 upd.Step();\r
199                 upd.Reset();\r
200                 st.Step();\r
201         }\r
202 \r
203         st.Finalize();\r
204         upd.Finalize();\r
205 \r
206         // insert all identities not in trust list already\r
207         m_db->Execute("INSERT INTO tblIdentityTrust(LocalIdentityID,IdentityID) SELECT LocalIdentityID,IdentityID FROM tblLocalIdentity,tblIdentity WHERE LocalIdentityID || '_' || IdentityID NOT IN (SELECT LocalIdentityID || '_' || IdentityID FROM tblIdentityTrust);");\r
208 \r
209         m_db->Execute("COMMIT;");\r
210 \r
211         m_log->debug("PeriodicDBMaintenance::Do1HourMaintenance");\r
212 }\r
213 \r
214 void DBMaintenanceThread::Do6HourMaintenance()\r
215 {\r
216 \r
217         m_db->Execute("BEGIN;");\r
218 \r
219         // if we remove a board and the reply boardid is still set to it, we need to replace it with a boardid that does exist\r
220         SQLite3DB::Statement st=m_db->Prepare("SELECT MessageID FROM tblMessage WHERE ReplyBoardID NOT IN (SELECT BoardID FROM tblBoard);");\r
221         SQLite3DB::Statement st2=m_db->Prepare("SELECT BoardID FROM tblMessageBoard WHERE MessageID=?;");\r
222         SQLite3DB::Statement upd=m_db->Prepare("UPDATE tblMessage SET ReplyBoardID=? WHERE MessageID=?;");\r
223         st.Step();\r
224         while(st.RowReturned())\r
225         {\r
226                 // find a valid boardid for the message\r
227                 int messageid=0;\r
228                 int boardid=0;\r
229 \r
230                 st.ResultInt(0,messageid);\r
231 \r
232                 st2.Bind(0,messageid);\r
233                 st2.Step();\r
234                 if(st2.RowReturned())\r
235                 {\r
236                         st2.ResultInt(0,boardid);\r
237                         upd.Bind(0,boardid);\r
238                         upd.Bind(1,messageid);\r
239                         upd.Step();\r
240                         upd.Reset();\r
241                 }\r
242                 st2.Reset();\r
243                 \r
244                 st.Step();\r
245         }\r
246 \r
247         st.Finalize();\r
248         st2.Finalize();\r
249         upd.Finalize();\r
250 \r
251         m_db->Execute("COMMIT;");\r
252 \r
253         m_log->debug("PeriodicDBMaintenance::Do6HourMaintenance");\r
254 }\r
255 \r
256 void DBMaintenanceThread::Do1DayMaintenance()\r
257 {\r
258         Poco::DateTime date;\r
259 \r
260         m_db->Execute("BEGIN;");\r
261 \r
262         // delete all puzzles 2 or more days old\r
263         date=Poco::Timestamp();\r
264         date-=Poco::Timespan(2,0,0,0,0);\r
265         m_db->Execute("DELETE FROM tblIntroductionPuzzleInserts WHERE Day<='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
266         m_db->Execute("DELETE FROM tblIntroductionPuzzleRequests WHERE Day<='"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
267 \r
268         // delete all identities we've never seen and were added more than 20 days ago\r
269         date=Poco::Timestamp();\r
270         date-=Poco::Timespan(20,0,0,0,0);\r
271         m_db->Execute("DELETE FROM tblIdentity WHERE LastSeen IS NULL AND DateAdded<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
272 \r
273         // delete old identity requests - we don't need them anymore\r
274         date=Poco::Timestamp();\r
275         date-=Poco::Timespan(2,0,0,0,0);\r
276         m_db->Execute("DELETE FROM tblIdentityRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
277 \r
278         // delete old board list inserts/requests - we don't need them anymore\r
279         date=Poco::Timestamp();\r
280         date-=Poco::Timespan(2,0,0,0,0);\r
281         m_db->Execute("DELETE FROM tblBoardListInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
282         m_db->Execute("DELETE FROM tblBoardListRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
283 \r
284         // delete old local identity inserts - we don't need them anymore\r
285         date=Poco::Timestamp();\r
286         date-=Poco::Timespan(2,0,0,0,0);\r
287         m_db->Execute("DELETE FROM tblLocalIdentityInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
288 \r
289         // delete old message list inserts/requests - we don't need them anymore\r
290         date=Poco::Timestamp();\r
291         date-=Poco::Timespan(2,0,0,0,0);\r
292         m_db->Execute("DELETE FROM tblMessageListInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
293         m_db->Execute("DELETE FROM tblMessageListRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
294 \r
295         // delete old trust list inserts/requests - we don't need them anymore\r
296         date=Poco::Timestamp();\r
297         date-=Poco::Timespan(2,0,0,0,0);\r
298         m_db->Execute("DELETE FROM tblTrustListInserts WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
299         m_db->Execute("DELETE FROM tblTrustListRequests WHERE Day<'"+Poco::DateTimeFormatter::format(date,"%Y-%m-%d")+"';");\r
300 \r
301         // delete trust lists from identities we aren't trusting anymore\r
302         m_db->Execute("DELETE FROM tblPeerTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity WHERE (LocalTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinLocalTrustListTrust')) AND (PeerTrustListTrust IS NULL OR PeerTrustListTrust>=(SELECT OptionValue FROM tblOption WHERE Option='MinPeerTrustListTrust')));");\r
303 \r
304         // remove identityid from messages where the identity has been deleted\r
305         m_db->Execute("UPDATE tblMessage SET IdentityID=NULL WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);");\r
306 \r
307         // try to re-attach messages from identities that were previously deleted, but have been since re-added\r
308         // first get the names from messages that have a NULL IdentityID\r
309         SQLite3DB::Statement st=m_db->Prepare("SELECT FromName FROM tblMessage WHERE IdentityID IS NULL GROUP BY FromName;");\r
310         SQLite3DB::Statement findst=m_db->Prepare("SELECT IdentityID,PublicKey FROM tblIdentity WHERE Name=?;");\r
311         st.Step();\r
312         while(st.RowReturned())\r
313         {\r
314                 std::string name="";\r
315                 std::string namepart="";\r
316                 std::string publickey="";\r
317                 int identityid=0;\r
318                 st.ResultText(0,name);\r
319 \r
320                 std::vector<std::string> parts;\r
321                 StringFunctions::Split(name,"@",parts);\r
322 \r
323                 // name can have a @ in it - so reattach all parts except the last which is the key\r
324                 for(long i=0; i<parts.size()-1; i++)\r
325                 {\r
326                         if(i!=0)\r
327                         {\r
328                                 namepart+="@";\r
329                         }\r
330                         namepart+=parts[i];\r
331                 }\r
332 \r
333                 // find identities with this name\r
334                 findst.Bind(0,namepart);\r
335                 findst.Step();\r
336                 while(findst.RowReturned())\r
337                 {\r
338                         publickey="";\r
339                         identityid=0;\r
340                         findst.ResultText(1,publickey);\r
341                         // check if public key matches 2nd part\r
342                         if(parts.size()>1 && publickey.find(parts[1])==4)\r
343                         {\r
344                                 // we have the identity - so update the messages table with the identityid\r
345                                 findst.ResultInt(0,identityid);\r
346 \r
347                                 SQLite3DB::Statement st3=m_db->Prepare("UPDATE tblMessage SET IdentityID=? WHERE FromName=? AND IdentityID IS NULL;");\r
348                                 st3.Bind(0,identityid);\r
349                                 st3.Bind(1,name);\r
350                                 st3.Step();\r
351                         }\r
352                         findst.Step();\r
353                 }\r
354                 findst.Reset();\r
355 \r
356                 st.Step();\r
357         }\r
358 \r
359         // delete single use identities that are older than 7 days\r
360         date=Poco::Timestamp();\r
361         date-=Poco::Timespan(7,0,0,0,0);\r
362         st=m_db->Prepare("DELETE FROM tblIdentity WHERE SingleUse='true' AND DateAdded<?;");\r
363         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S"));\r
364         st.Step();\r
365 \r
366         // delete local single use identities that are older than 7 days\r
367         date=Poco::Timestamp();\r
368         date-=Poco::Timespan(7,0,0,0,0);\r
369         st=m_db->Prepare("DELETE FROM tblLocalIdentity WHERE SingleUse='true' AND DateCreated<?;");\r
370         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d %H:%M:%S"));\r
371         st.Step();\r
372 \r
373         // delete old messages\r
374         date=Poco::Timestamp();\r
375         date-=Poco::Timespan(m_deletemessagesolderthan,0,0,0,0);\r
376         m_log->trace("PeriodicDBMaintenance::Do1DayMaintenance deleting messages prior to "+Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
377         st=m_db->Prepare("DELETE FROM tblMessage WHERE MessageDate<?;");\r
378         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
379         st.Step();\r
380 \r
381         // delete old message requests\r
382         date=Poco::Timestamp();\r
383         date-=Poco::Timespan(m_messagedownloadmaxdaysbackward,0,0,0,0);\r
384         st=m_db->Prepare("DELETE FROM tblMessageRequests WHERE Day<?;");\r
385         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
386         st.Step();\r
387 \r
388         // delete old frost message requests\r
389         date=Poco::Timestamp();\r
390         date-=Poco::Timespan(m_frostmaxdaysbackward,0,0,0,0);\r
391         st=m_db->Prepare("DELETE FROM tblFrostMessageRequests WHERE Day<?;");\r
392         st.Bind(0,Poco::DateTimeFormatter::format(date,"%Y-%m-%d"));\r
393         st.Step();\r
394 \r
395         // delete tblIdentityTrust for local identities and identities that have been deleted\r
396         m_db->Execute("DELETE FROM tblIdentityTrust WHERE LocalIdentityID NOT IN (SELECT LocalIdentityID FROM tblLocalIdentity);");\r
397         m_db->Execute("DELETE FROM tblIdentityTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);");\r
398 \r
399 \r
400         // cap failure count\r
401         m_db->Execute("UPDATE tblIdentity SET FailureCount=(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount') WHERE FailureCount>(SELECT OptionValue FROM tblOption WHERE Option='MaxFailureCount');");\r
402         // reduce failure count for each identity\r
403         m_db->Execute("UPDATE tblIdentity SET FailureCount=0 WHERE FailureCount<(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');");\r
404         m_db->Execute("UPDATE tblIdentity SET FailureCount=FailureCount-(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction') WHERE FailureCount>=(SELECT OptionValue FROM tblOption WHERE Option='FailureCountReduction');");\r
405 \r
406         st.Finalize();\r
407         findst.Finalize();\r
408 \r
409         m_db->Execute("COMMIT;");\r
410 \r
411         m_log->debug("PeriodicDBMaintenance::Do1DayMaintenance");\r
412 \r
413 }\r
414 \r
415 void DBMaintenanceThread::run()\r
416 {\r
417         m_log->debug("DBMaintenanceThread::run thread started.");\r
418 \r
419         LoadDatabase();\r
420         Option option(m_db);\r
421         std::string tempval("");\r
422 \r
423         m_deletemessagesolderthan=180;\r
424         tempval="180";\r
425         option.Get("DeleteMessagesOlderThan",tempval);\r
426         StringFunctions::Convert(tempval,m_deletemessagesolderthan);\r
427 \r
428         m_messagedownloadmaxdaysbackward=5;\r
429         tempval="5";\r
430         option.Get("MessageDownloadMaxDaysBackward",tempval);\r
431         StringFunctions::Convert(tempval,m_messagedownloadmaxdaysbackward);\r
432 \r
433         m_frostmaxdaysbackward=5;\r
434         tempval="5";\r
435         option.Get("FrostMessageMaxDaysBackward",tempval);\r
436         StringFunctions::Convert(tempval,m_frostmaxdaysbackward);\r
437 \r
438         Poco::DateTime now;\r
439         int i=0;\r
440 \r
441         do\r
442         {\r
443                 now=Poco::Timestamp();\r
444 \r
445                 if((m_last10minute+Poco::Timespan(0,0,10,0,0))<=now)\r
446                 {\r
447                         Do10MinuteMaintenance();\r
448                         m_last10minute=Poco::Timestamp();\r
449                 }\r
450                 /*\r
451                 if((m_last30minute+Poco::Timespan(0,0,30,0,0))<=now)\r
452                 {\r
453                         Do30MinuteMaintenance();\r
454                         m_last30minute=Poco::Timestamp();\r
455                 }\r
456                 */\r
457                 if((m_last1hour+Poco::Timespan(0,1,0,0,0))<=now)\r
458                 {\r
459                         Do1HourMaintenance();\r
460                         m_last1hour=Poco::Timestamp();\r
461                 }\r
462                 if((m_last6hour+Poco::Timespan(0,6,0,0,0))<=now)\r
463                 {\r
464                         Do6HourMaintenance();\r
465                         m_last6hour=Poco::Timestamp();\r
466                 }\r
467                 if((m_last1day+Poco::Timespan(1,0,0,0,0))<=now)\r
468                 {\r
469                         Do1DayMaintenance();\r
470                         m_last1day=Poco::Timestamp();\r
471                 }\r
472 \r
473                 i=0;\r
474                 while(i++<5 && !IsCancelled())\r
475                 {\r
476                         Poco::Thread::sleep(1000);\r
477                 }\r
478 \r
479         }while(!IsCancelled());\r
480 \r
481         m_log->debug("DBMaintenanceThread::run thread exiting.");\r
482 }\r