From bae54a5dae027ad0a6b0126ec9eeecd3bd2739eb Mon Sep 17 00:00:00 2001 From: SomeDude Date: Sun, 6 Apr 2008 09:04:00 +0200 Subject: [PATCH] version 0.2.8 --- CMakeLists.txt | 19 ++++++++- include/charsetconverter.h | 26 ++++++++++++ include/freenet/iindexinserter.h | 4 +- include/freenet/iindexrequester.h | 8 ++-- include/global.h | 3 +- readme.txt | 3 +- src/charsetconverter.cpp | 79 +++++++++++++++++++++++++++++++++++ src/freenet/periodicdbmaintenance.cpp | 4 ++ src/global.cpp | 2 + src/http/pages/peerdetailspage.cpp | 4 +- src/http/pages/peertrustpage.cpp | 4 +- src/message.cpp | 62 ++++++++++++++------------- 12 files changed, 176 insertions(+), 42 deletions(-) create mode 100644 include/charsetconverter.h create mode 100644 src/charsetconverter.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c81d5cb..742bfc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,7 @@ PROJECT(fms) -OPTION(USE_BUNDLED_SQLITE "Use the bundled SQLite3 library" OFF) +OPTION(USE_BUNDLED_SQLITE "Use the bundled SQLite3 library." OFF) +OPTION(DO_CHARSET_CONVERSION "Do charset conversion on sent messages to UTF-8. This requires libiconv." OFF) SET(FMS_SRC src/base64.cpp @@ -84,6 +85,10 @@ src/pthreadwrapper/threadedexecutor.cpp src/xyssl/sha1.c ) +IF(DO_CHARSET_CONVERSION) + SET(FMS_SRC ${FMS_SRC} src/charsetconverter.cpp) +ENDIF(DO_CHARSET_CONVERSION) + IF(WIN32) SET(FMS_PLATFORM_SRC src/fmsservice.cpp) ELSE(WIN32) @@ -132,6 +137,7 @@ ENDIF(NOT USE_BUNDLED_SQLITE) FIND_LIBRARY(TINYXML_LIBRARY NAMES tinyxml tinyxml_s) FIND_LIBRARY(PTHREADS_LIBRARY NAMES pthread pthreads pthreadvc2) FIND_LIBRARY(SHTTPD_LIBRARY NAMES shttpd shttpd_s) +FIND_LIBRARY(ICONV_LIBRARY NAMES iconv iconv_s libiconv libiconv_s) IF(SQLITE3_LIBRARY) TARGET_LINK_LIBRARIES(fms ${SQLITE3_LIBRARY}) @@ -174,3 +180,14 @@ ELSE(SHTTPD_LIBRARY) ADD_LIBRARY(shttpd ${SHTTPD_PLATFORM_SRC} libs/shttpd/auth.c libs/shttpd/cgi.c libs/shttpd/config.c libs/shttpd/io_cgi.c libs/shttpd/io_dir.c libs/shttpd/io_emb.c libs/shttpd/io_file.c libs/shttpd/io_socket.c libs/shttpd/io_ssi.c libs/shttpd/io_ssl.c libs/shttpd/log.c libs/shttpd/md5.c libs/shttpd/shttpd.c libs/shttpd/string.c) TARGET_LINK_LIBRARIES(fms shttpd) ENDIF(SHTTPD_LIBRARY) + +IF(DO_CHARSET_CONVERSION) + ADD_DEFINITIONS(-DDO_CHARSET_CONVERSION) + IF(ICONV_LIBRARY) + TARGET_LINK_LIBRARIES(fms ${ICONV_LIBRARY}) + ELSE(ICONV_LIBRARY) + IF(WIN32) + MESSAGE(FATAL ERROR "Could not find iconv library. You must set the location manually, or turn off charset conversion.") + ENDIF(WIN32) + ENDIF(ICONV_LIBRARY) +ENDIF(DO_CHARSET_CONVERSION) diff --git a/include/charsetconverter.h b/include/charsetconverter.h new file mode 100644 index 0000000..61209d8 --- /dev/null +++ b/include/charsetconverter.h @@ -0,0 +1,26 @@ +#ifndef _charset_converter_ +#define _charset_converter_ + +#include +#include + +class CharsetConverter +{ +public: + CharsetConverter(); + CharsetConverter(const std::string &fromcharset, const std::string &tocharset); + ~CharsetConverter(); + + const bool SetConversion(const std::string &fromcharset, const std::string &tocharset); + + const bool Convert(const std::string &input, std::string &output); + +private: + + iconv_t m_iconv; + std::string m_fromcharset; + std::string m_tocharset; + +}; + +#endif // _charset_converter_ diff --git a/include/freenet/iindexinserter.h b/include/freenet/iindexinserter.h index 050aa3d..035c5f3 100644 --- a/include/freenet/iindexinserter.h +++ b/include/freenet/iindexinserter.h @@ -66,11 +66,11 @@ void IIndexInserter::FCPConnected() // make sure variables have been initialized by the derived class if(m_fcpuniquename=="") { - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexInserter::FCPConnected fcpuniquename not initialized correctly!"); + m_log->WriteLog(LogFile::LOGLEVEL_FATAL,"IIndexInserter::FCPConnected fcpuniquename not initialized correctly!"); } if(m_fcpuniquename.find("|")!=std::string::npos) { - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexInserter::FCPConnected fcpuniquename contains | character! This is not a valid character!"); + m_log->WriteLog(LogFile::LOGLEVEL_FATAL,"IIndexInserter::FCPConnected fcpuniquename contains | character! This is not a valid character!"); } m_inserting.clear(); diff --git a/include/freenet/iindexrequester.h b/include/freenet/iindexrequester.h index aecc00a..8744dfd 100644 --- a/include/freenet/iindexrequester.h +++ b/include/freenet/iindexrequester.h @@ -71,15 +71,15 @@ void IIndexRequester::FCPConnected() // make sure variables have been initialized by the derived class if(m_maxrequests==-1) { - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexRequester::FCPConnected maxrequests not initialized correctly!"); + m_log->WriteLog(LogFile::LOGLEVEL_FATAL,"IIndexRequester::FCPConnected maxrequests not initialized correctly!"); } if(m_fcpuniquename=="") { - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexRequester::FCPConnected fcpuniquename not initialized correctly!"); + m_log->WriteLog(LogFile::LOGLEVEL_FATAL,"IIndexRequester::FCPConnected fcpuniquename not initialized correctly!"); } if(m_fcpuniquename.find("|")!=std::string::npos) { - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexRequester::FCPConnected fcpuniquename contains | character! This is not a valid character!"); + m_log->WriteLog(LogFile::LOGLEVEL_FATAL,"IIndexRequester::FCPConnected fcpuniquename contains | character! This is not a valid character!"); } m_lastreceived.SetToGMTime(); @@ -189,7 +189,7 @@ void IIndexRequester::Process() // if we haven't received any messages to this object in 10 minutes, clear the requests and repopulate id list if(m_ids.size()>0 && m_lastreceived<(now-(1.0/144.0))) { - m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexRequester::Process() Object has not received any messages in 10 minutes. Restarting requests."); + m_log->WriteLog(LogFile::LOGLEVEL_ERROR,"IIndexRequester::Process "+m_fcpuniquename+" Object has not received any messages in 10 minutes. Restarting requests."); FCPConnected(); } diff --git a/include/global.h b/include/global.h index 834f605..c5f7325 100644 --- a/include/global.h +++ b/include/global.h @@ -1,10 +1,11 @@ #ifndef _global_ #define _global_ +#include #include #include "pthreadwrapper/thread.h" -#define FMS_VERSION "0.2.7" +#define FMS_VERSION "0.2.8" // opens database and creates tables and initial inserts if necessary void SetupDB(); diff --git a/readme.txt b/readme.txt index db879dc..be55508 100644 --- a/readme.txt +++ b/readme.txt @@ -8,7 +8,8 @@ cmake . make If you want to use the bundled SQLite3 library, add a -D USE_BUNDLED_SQLITE=ON -to the cmake command. +to the cmake command. To do charset conversion to UTF-8 when sending messages, +add a -D DO_CHARSET_CONVERSION=ON. UPGRADING --------- diff --git a/src/charsetconverter.cpp b/src/charsetconverter.cpp new file mode 100644 index 0000000..f6f73d4 --- /dev/null +++ b/src/charsetconverter.cpp @@ -0,0 +1,79 @@ +#include "../include/charsetconverter.h" + +#include + +CharsetConverter::CharsetConverter() +{ + m_iconv=(iconv_t)-1; + m_fromcharset=""; + m_tocharset=""; +} + +CharsetConverter::CharsetConverter(const std::string &fromcharset, const std::string &tocharset) +{ + SetConversion(fromcharset,tocharset); +} + +CharsetConverter::~CharsetConverter() +{ + if(m_iconv!=(iconv_t)-1) + { + iconv_close(m_iconv); + } +} + +const bool CharsetConverter::SetConversion(const std::string &fromcharset, const std::string &tocharset) +{ + if(m_iconv!=(iconv_t)-1) + { + iconv_close(m_iconv); + m_iconv=(iconv_t)-1; + } + if((m_iconv=iconv_open(tocharset.c_str(),fromcharset.c_str()))!=(iconv_t)-1) + { + m_fromcharset=fromcharset; + m_tocharset=tocharset; + return true; + } + else + { + m_fromcharset=""; + m_tocharset=""; + } +} + +const bool CharsetConverter::Convert(const std::string &input, std::string &output) +{ + if(m_iconv!=(iconv_t)-1) + { + std::vector invec(input.begin(),input.end()); + std::vector outvec(invec.size()*4,0); +#ifdef _WIN32 + const char *inptr=&invec[0]; +#else + char *inptr=&invec[0]; +#endif + char *outptr=&outvec[0]; + size_t insize=invec.size(); + size_t outsize=outvec.size(); + size_t rval=0; + + rval=iconv(m_iconv,&inptr,&insize,&outptr,&outsize); + + if(outsize>=0) + { + outvec.resize(outptr-&outvec[0]); + output=""; + output.append(outvec.begin(),outvec.end()); + } + else + { + return false; + } + + } + else + { + return false; + } +} diff --git a/src/freenet/periodicdbmaintenance.cpp b/src/freenet/periodicdbmaintenance.cpp index 84e7e94..073da19 100644 --- a/src/freenet/periodicdbmaintenance.cpp +++ b/src/freenet/periodicdbmaintenance.cpp @@ -245,6 +245,10 @@ void PeriodicDBMaintenance::Do1DayMaintenance() st.Bind(0,date.Format("%Y-%m-%d")); st.Step(); + // delete tblIdentityTrust for local identities and identities that have been deleted + m_db->Execute("DELETE FROM tblIdentityTrust WHERE LocalIdentityID NOT IN (SELECT LocalIdentityID FROM tblLocalIdentity);"); + m_db->Execute("DELETE FROM tblIdentityTrust WHERE IdentityID NOT IN (SELECT IdentityID FROM tblIdentity);"); + } void PeriodicDBMaintenance::Process() diff --git a/src/global.cpp b/src/global.cpp index 7889810..d145a54 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -420,6 +420,7 @@ void SetupDB() DELETE FROM tblMessageReplyTo WHERE tblMessageReplyTo.MessageID=old.MessageID;\ END;"); + db->Execute("DROP TRIGGER IF EXISTS trgDeleteIdentity;"); db->Execute("CREATE TRIGGER IF NOT EXISTS trgDeleteIdentity AFTER DELETE ON tblIdentity \ FOR EACH ROW \ BEGIN \ @@ -429,6 +430,7 @@ void SetupDB() DELETE FROM tblMessageRequests WHERE IdentityID=old.IdentityID;\ DELETE FROM tblPeerTrust WHERE IdentityID=old.IdentityID;\ DELETE FROM tblTrustListRequests WHERE IdentityID=old.IdentityID;\ + DELETE FROM tblIdentityTrust WHERE IdentityID=old.IdentityID;\ END;"); db->Execute("DROP TRIGGER IF EXISTS trgDeleteLocalIdentity;"); diff --git a/src/http/pages/peerdetailspage.cpp b/src/http/pages/peerdetailspage.cpp index 78ea02c..5aa8006 100644 --- a/src/http/pages/peerdetailspage.cpp +++ b/src/http/pages/peerdetailspage.cpp @@ -69,8 +69,10 @@ const std::string PeerDetailsPage::GeneratePage(const std::string &method, const usk=publickey; if(usk.find("SSK@")==0) { + std::string messagebase=""; + Option::Instance()->Get("MessageBase",messagebase); usk.erase(0,3); - usk="USK"+usk+"fms/0/"; + usk="USK"+usk+messagebase+"/0/"; } content+="Name"+SanitizeOutput(name)+""; diff --git a/src/http/pages/peertrustpage.cpp b/src/http/pages/peertrustpage.cpp index 21b0730..dc689bf 100644 --- a/src/http/pages/peertrustpage.cpp +++ b/src/http/pages/peertrustpage.cpp @@ -131,7 +131,7 @@ const std::string PeerTrustPage::GeneratePage(const std::string &method, const s } } - if(queryvars.find("formaction")!=queryvars.end() && (*queryvars.find("formaction")).second=="update") + if(localidentityid!=-1 && queryvars.find("formaction")!=queryvars.end() && (*queryvars.find("formaction")).second=="update") { std::vector identityids; std::vector oldlmt; @@ -415,7 +415,7 @@ const std::string PeerTrustPage::GeneratePage(const std::string &method, const s content+=""; } - content+="
"; + content+="
"; content+=""; content+=""; diff --git a/src/message.cpp b/src/message.cpp index eea0633..8582146 100644 --- a/src/message.cpp +++ b/src/message.cpp @@ -7,6 +7,10 @@ #include +#ifdef DO_CHARSET_CONVERSION + #include "../include/charsetconverter.h" +#endif + #ifdef XMEM #include #endif @@ -611,6 +615,19 @@ const bool Message::ParseNNTPMessage(const std::string &nntpmessage) m_subject=mime.GetFieldValue("Subject"); // remove any path folding m_subject=StringFunctions::Replace(m_subject,"\r\n",""); +#if DO_CHARSET_CONVERSION + if(mime.GetFieldCharset("Subject")) + { + std::string charset=mime.GetFieldCharset("Subject"); + CharsetConverter ccv; + if(charset!="" && charset!="UTF-8" && ccv.SetConversion(charset,"UTF-8")) + { + std::string output=""; + ccv.Convert(m_subject,output); + m_subject=output; + } + } +#endif } else { @@ -658,7 +675,21 @@ const bool Message::ParseNNTPMessage(const std::string &nntpmessage) { if((*i)->IsText() && (*i)->GetContent()) { - m_body+=(char *)(*i)->GetContent(); + std::string bodypart=(char *)(*i)->GetContent(); +#ifdef DO_CHARSET_CONVERSION + std::string charset=(*i)->GetCharset(); + if(charset!="" && charset!="UTF-8") + { + CharsetConverter ccv; + if(ccv.SetConversion(charset,"UTF-8")) + { + std::string output=""; + ccv.Convert(bodypart,output); + bodypart=output; + } + } +#endif + m_body+=bodypart; } } @@ -689,35 +720,6 @@ const bool Message::StartFreenetInsert() xml.AddInReplyTo((*j).first,(*j).second); } - - // find identity to insert with - /* - SQLite3DB::Statement st=m_db->Prepare("SELECT LocalIdentityID FROM tblLocalIdentity WHERE Name=?;"); - st.Bind(0,m_fromname); - st.Step(); - - // couldn't find identity with this name - insert a new identity - if(!st.RowReturned()) - { - if(m_addnewpostfromidentities==true) - { - DateTime now; - now.SetToGMTime(); - st=m_db->Prepare("INSERT INTO tblLocalIdentity(Name) VALUES(?);"); - st.Bind(0,m_fromname); - st.Step(true); - localidentityid=st.GetLastInsertRowID(); - } - else - { - return false; - } - } - else - { - st.ResultInt(0,localidentityid); - } - */ localidentityid=FindLocalIdentityID(m_fromname); if(localidentityid==-1) { -- 2.7.4