Adds iconv implementation to convert between input charset and UTF-8 for display.

5 files changed, 209 insertions, 15 deletions

diff --git a/src/about/updates.gemini b/src/about/updates.gemini
index 6b4ef87..615300e 100644
--- a/src/about/updates.gemini
+++ b/src/about/updates.gemini
@@ -12,7 +12,8 @@
 * Client certificates are disabled when doing a host switch
 * Redirection handling is now configurable and contains some warning messages for potentially malicious redirects.
 gr
-
+* Updated application icon thanks to tiwesdaeg
+* Survives conman's client torture suite
 
 ## 0.2 - The protocol update
 * Implement Ctrl+D/*Add to favourites* menu item
diff --git a/src/browsertab.cpp b/src/browsertab.cpp
index 0733f53..1738483 100644
--- a/src/browsertab.cpp
+++ b/src/browsertab.cpp
@@ -7,6 +7,8 @@
 #include "geminirenderer.hpp"
 #include "plaintextrenderer.hpp"
 
+#include "mimeparser.hpp"
+
 #include "certificateselectiondialog.hpp"
 
 #include "geminiclient.hpp"
@@ -36,6 +38,8 @@
 #include <QGraphicsPixmapItem>
 #include <QGraphicsTextItem>
 
+#include <iconv.h>
+
 BrowserTab::BrowserTab(MainWindow *mainWindow) : QWidget(nullptr),
                                                  ui(new Ui::BrowserTab),
                                                  mainWindow(mainWindow),
@@ -228,11 +232,104 @@ void BrowserTab::on_certificateRequired(const QString &reason)
     this->updateUI();
 }
 
-void BrowserTab::on_requestComplete(const QByteArray &data, const QString &mime)
+static QByteArray convertToUtf8(QByteArray const & input, QString const & charSet)
+{
+    QFile temp { "/tmp/raw.dat" };
+    temp.open(QFile::WriteOnly);
+    IoUtil::writeAll(temp, input);
+
+    auto charset_u8 = charSet.toUpper().toUtf8();
+
+    // TRANSLIT will try to mix-match other code points to reflect to correct encoding
+    iconv_t cd = iconv_open("UTF-8", charset_u8.data());
+    if(cd == (iconv_t)-1) {
+        return QByteArray { };
+    }
+
+    QByteArray result;
+
+    char temp_buffer[4096];
+
+    char const * input_ptr = reinterpret_cast<char const *>(input.data());
+    size_t input_size = input.size();
+
+    while(input_size > 0)
+    {
+        char * out_ptr = temp_buffer;
+        size_t out_size = sizeof(temp_buffer);
+
+        size_t n = iconv(cd, const_cast<char **>(&input_ptr), &input_size, &out_ptr, &out_size);
+        if (n == size_t(-1))
+        {
+            if(errno == E2BIG) {
+                // silently ignore E2BIG, as we will continue conversion in the next loop
+            }
+            else if(errno == EILSEQ) {
+                // this is an invalid multibyte sequence.
+                // append an "replacement character" and skip a byte
+                if(input_size > 0) {
+                    input_size --;
+                    input_ptr++;
+                    result.append(u8"�");
+                }
+            }
+            else if(errno == EINVAL) {
+                // the file ends with an invalid multibyte sequence.
+                // just drop it and display the replacement-character
+                if(input_size > 0) {
+                    input_size --;
+                    input_ptr++;
+                    result.append(u8"�");
+                }
+            }
+            else {
+                perror("iconv conversion error");
+                break;
+            }
+        }
+
+        size_t len = out_ptr - temp_buffer;
+        result.append(temp_buffer, len);
+    }
+
+    iconv_close(cd);
+
+    return result;
+}
+
+void BrowserTab::on_requestComplete(const QByteArray &ref_data, const QString &mime_text)
 {
-    qDebug() << "Loaded" << data.length() << "bytes of type" << mime;
+    QByteArray data = ref_data;
+    MimeType mime = MimeParser::parse(mime_text);
+
+    qDebug() << "Loaded" << ref_data.length() << "bytes of type" << mime.type << "/" << mime.subtype;
+//    for(auto & key : mime.parameters.keys()) {
+//        qDebug() << key << mime.parameters[key];
+//    }
+
+    auto charset = mime.parameter("charset", "utf-8").toUpper();
+    if(not ref_data.isEmpty() and (mime.type == "text") and (charset != "UTF-8"))
+    {
+        auto temp = convertToUtf8(ref_data, charset);
+        bool ok = (temp.size() > 0);
+        if(ok) {
+            data = std::move(temp);
+        } else {
+            auto response = QMessageBox::question(
+                this,
+                "Kristall",
+                QString("Failed to convert input charset %1 to UTF-8. Cannot display the file.\r\nDo you want to display unconverted data anyways?").arg(charset)
+            );
+
+            if(response != QMessageBox::Yes) {
+                setErrorMessage(QString("Failed to convert input charset %1 to UTF-8.").arg(charset));
+                return;
+            }
+        }
+    }
+
 
-    this->current_mime = mime;
+    this->current_mime = mime_text;
     this->current_buffer = data;
 
     this->graphics_scene.clear();
@@ -258,7 +355,7 @@ void BrowserTab::on_requestComplete(const QByteArray &data, const QString &mime)
 
     bool plaintext_only = (global_options.text_display == GenericSettings::PlainText);
 
-    if (not plaintext_only and mime.startsWith("text/gemini"))
+    if (not plaintext_only and mime_text.startsWith("text/gemini"))
     {
         document = GeminiRenderer::render(
             data,
@@ -266,18 +363,18 @@ void BrowserTab::on_requestComplete(const QByteArray &data, const QString &mime)
             doc_style,
             this->outline);
     }
-    else if (not plaintext_only and mime.startsWith("text/gophermap"))
+    else if (not plaintext_only and mime_text.startsWith("text/gophermap"))
     {
         document = GophermapRenderer::render(
             data,
             this->current_location,
             doc_style);
     }
-    else if (not plaintext_only and mime.startsWith("text/finger"))
+    else if (not plaintext_only and mime_text.startsWith("text/finger"))
     {
         document = PlainTextRenderer::render(data, doc_style);
     }
-    else if (not plaintext_only and mime.startsWith("text/html"))
+    else if (not plaintext_only and mime_text.startsWith("text/html"))
     {
         document = std::make_unique<QTextDocument>();
 
@@ -287,7 +384,7 @@ void BrowserTab::on_requestComplete(const QByteArray &data, const QString &mime)
         document->setHtml(QString::fromUtf8(data));
     }
 #if defined(QT_FEATURE_textmarkdownreader)
-    else if (not plaintext_only and mime.startsWith("text/markdown"))
+    else if (not plaintext_only and mime_text.startsWith("text/markdown"))
     {
         document = std::make_unique<QTextDocument>();
         document->setDefaultFont(doc_style.standard_font);
@@ -296,11 +393,11 @@ void BrowserTab::on_requestComplete(const QByteArray &data, const QString &mime)
         document->setMarkdown(QString::fromUtf8(data));
     }
 #endif
-    else if (mime.startsWith("text/"))
+    else if (mime_text.startsWith("text/"))
     {
         document = PlainTextRenderer::render(data, doc_style);
     }
-    else if (mime.startsWith("image/"))
+    else if (mime_text.startsWith("image/"))
     {
         doc_type = Image;
 
@@ -333,10 +430,10 @@ void BrowserTab::on_requestComplete(const QByteArray &data, const QString &mime)
 
         this->ui->graphics_browser->fitInView(graphics_scene.sceneRect(), Qt::KeepAspectRatio);
     }
-    else if (mime.startsWith("video/") or mime.startsWith("audio/"))
+    else if (mime_text.startsWith("video/") or mime_text.startsWith("audio/"))
     {
         doc_type = Media;
-        this->ui->media_browser->setMedia(data, this->current_location, mime);
+        this->ui->media_browser->setMedia(data, this->current_location, mime_text);
     }
     else
     {
@@ -352,7 +449,7 @@ Info:
 MIME Type: %1
 File Size: %2
 )md")
-                                   .arg(mime)
+                                   .arg(mime_text)
                                    .arg(IoUtil::size_human(data.size())));
     }
 
@@ -370,7 +467,7 @@ File Size: %2
     QString title = this->current_location.toString();
     emit this->titleChanged(title);
 
-    emit this->fileLoaded(data.size(), mime, this->timer.elapsed());
+    emit this->fileLoaded(data.size(), mime_text, this->timer.elapsed());
 
     this->successfully_loaded = true;
 
diff --git a/src/kristall.pro b/src/kristall.pro
index 1cc2b86..38503dd 100644
--- a/src/kristall.pro
+++ b/src/kristall.pro
@@ -66,6 +66,7 @@ SOURCES += \
     main.cpp \
     mainwindow.cpp \
     mediaplayer.cpp \
+    mimeparser.cpp \
     newidentitiydialog.cpp \
     plaintextrenderer.cpp \
     protocolhandler.cpp \
@@ -101,6 +102,7 @@ HEADERS += \
     kristall.hpp \
     mainwindow.hpp \
     mediaplayer.hpp \
+    mimeparser.hpp \
     newidentitiydialog.hpp \
     plaintextrenderer.hpp \
     protocolhandler.hpp \
diff --git a/src/mimeparser.cpp b/src/mimeparser.cpp
new file mode 100644
index 0000000..dd254a4
--- /dev/null
+++ b/src/mimeparser.cpp
@@ -0,0 +1,63 @@
+#include "mimeparser.hpp"
+
+
+
+bool MimeType::is(const QString &type, const QString &sub_type) const
+{
+    return (this->type == type) and (this->subtype == sub_type);
+}
+
+QString MimeType::parameter(const QString &param_name, QString const & default_value) const
+{
+    auto val = parameters.value(param_name.toLower());
+    if(val.isNull())
+        val = default_value;
+    return val;
+}
+
+MimeType MimeParser::parse(const QString &mime_text)
+{
+    MimeType type;
+
+    QString arg_list;
+    QString mime_part;
+
+    if(int idx = mime_text.indexOf(' '); idx >= 0) {
+        arg_list = mime_text.mid(idx + 1).trimmed().toLower();
+        mime_part = mime_text.mid(0, idx).trimmed().toLower();
+    } else {
+        mime_part = mime_text.trimmed().toLower();
+    }
+
+    if(int idx = mime_part.indexOf('/'); idx >= 0) {
+        type.type = mime_part.mid(0, idx);
+        type.subtype = mime_part.mid(idx + 1);
+    } else {
+        type.type = mime_part;
+        type.subtype = QString { };
+    }
+
+    if(not arg_list.isEmpty()) {
+        for(auto const & _arg : arg_list.split(';'))
+        {
+            QString arg = _arg.trimmed();
+            if(arg.isEmpty()) // skip over double spaces
+                continue;
+
+            QString key;
+            QString value;
+
+            if(int idx = arg.indexOf('='); idx >= 0) {
+                key = arg.mid(0, idx);
+                value = arg.mid(idx + 1);
+            } else {
+                key = arg;
+                value = "";
+            }
+
+            type.parameters.insert(key, value);
+        }
+    }
+
+    return type;
+}
diff --git a/src/mimeparser.hpp b/src/mimeparser.hpp
new file mode 100644
index 0000000..d76594b
--- /dev/null
+++ b/src/mimeparser.hpp
@@ -0,0 +1,31 @@
+#ifndef MIMEPARSER_HPP
+#define MIMEPARSER_HPP
+
+#include <QMap>
+#include <QString>
+
+struct MimeType
+{
+    QString type;
+    QString subtype;
+    QMap<QString, QString> parameters;
+
+    bool is(QString const & type, QString const & sub_type) const;
+
+    QString parameter(QString const & param_name, QString const & default_value = QString { }) const;
+
+    bool isValid() const {
+        return not type.isEmpty();
+    }
+};
+
+struct MimeParser
+{
+    MimeParser() = delete;
+
+
+
+    static MimeType parse(QString const & mime_text);
+};
+
+#endif // MIMEPARSER_HPP