diff --git a/Telegram/SourceFiles/data/data_document_resolver.cpp b/Telegram/SourceFiles/data/data_document_resolver.cpp index 4bfda4ed75..79c1669928 100644 --- a/Telegram/SourceFiles/data/data_document_resolver.cpp +++ b/Telegram/SourceFiles/data/data_document_resolver.cpp @@ -17,9 +17,13 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "data/data_document_media.h" #include "data/data_file_click_handler.h" #include "data/data_session.h" +#include "history/view/media/history_view_gif.h" #include "history/history.h" #include "history/history_item.h" -#include "history/view/media/history_view_gif.h" +#ifdef TDESKTOP_NATIVE_MARKDOWN_IV +#include "iv/markdown/iv_markdown_common.h" +#include "iv/markdown/iv_markdown_controller.h" +#endif #include "lang/lang_keys.h" #include "media/player/media_player_instance.h" #include "platform/platform_file_utilities.h" @@ -29,6 +33,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "ui/widgets/checkbox.h" #include "ui/wrap/slide_wrap.h" #include "window/window_session_controller.h" + #include "styles/style_layers.h" #include @@ -247,8 +252,16 @@ void ResolveDocument( } else { document->saveFromDataSilent(); if (!openImageInApp()) { - if (!document->filepath(true).isEmpty()) { - LaunchWithWarning(location.name(), item); + const auto path = document->filepath(true); + if (!path.isEmpty()) { +#ifdef TDESKTOP_NATIVE_MARKDOWN_IV + const auto fileName = QFileInfo(path).fileName(); + if (Iv::Markdown::LooksLikeMarkdownFile(fileName) + && Iv::Markdown::TryOpenLocalFile(path)) { + return; + } +#endif + LaunchWithWarning(path, item); } else if (document->status == FileReady || document->status == FileDownloadFailed) { DocumentSaveClickHandler::Save( diff --git a/Telegram/SourceFiles/iv/markdown/README.md b/Telegram/SourceFiles/iv/markdown/README.md index 15787daa77..53851a3338 100644 --- a/Telegram/SourceFiles/iv/markdown/README.md +++ b/Telegram/SourceFiles/iv/markdown/README.md @@ -9,4 +9,4 @@ cmake -S . -B out -D TDESKTOP_NATIVE_MARKDOWN_IV=ON cmake --build out --config Debug --target Telegram ``` -The current skeleton is inert. It adds API and source structure plus dependency linkage only, with no file-opening hook, UI entry point, WebView Instant View change, localization, settings, or real Markdown/MicroTeX rendering. +The gated state now builds a cmark-gfm parser adapter, a value document model, a deterministic debug dump, and math extraction metadata. It also intercepts already-local `.md` and `.markdown` files and opens a minimal native diagnostic window. Full native Markdown rendering and MicroTeX output are still future work. diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.cpp b/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.cpp index 535358b878..2655f0cf68 100644 --- a/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.cpp +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.cpp @@ -1,15 +1,284 @@ #include "iv/markdown/iv_markdown_controller.h" +#include "iv/markdown/iv_markdown_parse.h" +#include "iv/markdown/iv_markdown_view.h" +#include "iv/iv_delegate_impl.h" +#include "logs.h" +#include "ui/widgets/rp_window.h" + +#include "styles/palette.h" +#include "styles/style_window.h" + +#include +#include +#include +#include + +#include +#include #include +#include namespace Iv::Markdown { +namespace { -Controller::Controller(OpenOptions options) -: _options(std::move(options)) { +constexpr auto kMaxSourceBytes = 4 * 1024 * 1024; + +[[nodiscard]] bool IsReadableLocalFile(const QFileInfo &info) { + return info.exists() && info.isFile() && info.isReadable(); } -const OpenOptions &Controller::options() const { - return _options; +[[nodiscard]] bool ReadLocalSource(const QString &path, QByteArray *bytes) { + const auto info = QFileInfo(path); + if (!IsReadableLocalFile(info)) { + return false; + } + auto file = QFile(path); + if (!file.open(QIODevice::ReadOnly)) { + return false; + } + const auto data = file.readAll(); + if (file.error() != QFileDevice::NoError) { + return false; + } + if (bytes) { + *bytes = data; + } + return true; +} + +[[nodiscard]] bool HasUtf8Bom(const QByteArray &source) { + return source.size() >= 3 + && static_cast(source[0]) == 0xEF + && static_cast(source[1]) == 0xBB + && static_cast(source[2]) == 0xBF; +} + +[[nodiscard]] unsigned char ByteAt(const QByteArray &source, int index) { + return static_cast(source[index]); +} + +[[nodiscard]] bool IsAllowedControl(unsigned char byte) { + return byte == '\t' || byte == '\n' || byte == '\r'; +} + +[[nodiscard]] bool LooksBinaryOrNonText(const QByteArray &source) { + const auto normalized = HasUtf8Bom(source) ? source.mid(3) : source; + const auto size = normalized.size(); + if (size == 0) { + return false; + } + auto controlBytes = 0; + for (auto i = 0; i != size; ++i) { + const auto byte = ByteAt(normalized, i); + if (byte == 0) { + return true; + } else if ((byte < 0x20 || byte == 0x7F) + && !IsAllowedControl(byte)) { + ++controlBytes; + } + } + return (controlBytes * 10) > size; +} + +[[nodiscard]] bool HasPreviewableContent(const MarkdownNode &node) { + switch (node.kind) { + case NodeKind::Document: + case NodeKind::Unsupported: + break; + default: + return true; + } + return std::any_of( + node.children.begin(), + node.children.end(), + [](const MarkdownNode &child) { + return HasPreviewableContent(child); + }); +} + +[[nodiscard]] bool AcceptsPreview(const PreparedDocument &document) { + return HasPreviewableContent(document.document) + || !document.formulas.empty(); +} + +class Controller final { +public: + Controller(PreparedDocument document, QString title); + + void activate(); + +private: + void close(); + void createWindow(); + void finishClose(); + + PreparedDocument _document; + QString _title; + Iv::DelegateImpl _delegate; + std::unique_ptr _window; + std::unique_ptr _preview; + bool _closing = false; + +}; + +[[nodiscard]] auto &ActiveControllers() { + static auto controllers = std::vector>(); + return controllers; +} + +void RemoveController(Controller *controller) { + auto &active = ActiveControllers(); + const auto i = std::find_if( + active.begin(), + active.end(), + [=](const std::unique_ptr &value) { + return value.get() == controller; + }); + if (i != active.end()) { + active.erase(i); + } +} + +void OpenDocumentWindow(PreparedDocument document, QString title) { + auto controller = std::make_unique( + std::move(document), + std::move(title)); + const auto raw = controller.get(); + ActiveControllers().push_back(std::move(controller)); + raw->activate(); +} + +Controller::Controller(PreparedDocument document, QString title) +: _document(std::move(document)) +, _title(std::move(title)) { + createWindow(); +} + +void Controller::activate() { + if (_window->isMinimized()) { + _window->showNormal(); + } else if (_window->isHidden()) { + _window->show(); + } + _window->raise(); + _window->activateWindow(); + _window->setFocus(); + if (_preview) { + _preview->setFocus(); + } +} + +void Controller::close() { + if (_window) { + _window->close(); + } +} + +void Controller::createWindow() { + _window = std::make_unique(); + const auto window = _window.get(); + window->setTitle(_title); + window->setWindowTitle(_title); + window->setGeometry(_delegate.ivGeometry(window)); + window->setMinimumSize({ st::windowMinWidth, st::windowMinHeight }); + window->geometryValue( + ) | rpl::distinct_until_changed( + ) | rpl::skip(1) | rpl::on_next([=] { + _delegate.ivSaveGeometry(window); + }, window->lifetime()); + window->body()->paintRequest() | rpl::on_next([=](QRect clip) { + QPainter(window->body().get()).fillRect(clip, st::windowBg); + }, window->body()->lifetime()); + + _preview = CreateMarkdownPreviewWidget( + _document, + OpenOptions{ .sourceName = _title }); + _preview->setParent(window->body().get()); + _preview->setGeometry(QRect(QPoint(), window->body()->size())); + window->body()->sizeValue() | rpl::on_next([=](QSize size) { + _preview->setGeometry(QRect(QPoint(), size)); + }, _preview->lifetime()); + _preview->show(); + + window->events() | rpl::on_next([=](not_null e) { + if (e->type() == QEvent::Close) { + finishClose(); + } else if (e->type() == QEvent::KeyPress) { + const auto event = static_cast(e.get()); + if (event->key() == Qt::Key_Escape) { + event->accept(); + close(); + } + } + }, window->lifetime()); + + window->show(); +} + +void Controller::finishClose() { + if (_closing) { + return; + } + _closing = true; + crl::on_main([controller = this] { + RemoveController(controller); + }); +} + +} // namespace + +bool TryOpenLocalFile(const QString &path) { + const auto info = QFileInfo(path); + if (!IsReadableLocalFile(info)) { + return false; + } + if (info.size() > kMaxSourceBytes) { + DEBUG_LOG(("Native Markdown IV: rejected local file too large: %1" + ).arg(path)); + return false; + } + + auto bytes = QByteArray(); + if (!ReadLocalSource(path, &bytes)) { + return false; + } + if (bytes.size() > kMaxSourceBytes) { + DEBUG_LOG(("Native Markdown IV: rejected local file too large: %1" + ).arg(path)); + return false; + } + if (LooksBinaryOrNonText(bytes)) { + DEBUG_LOG(("Native Markdown IV: rejected local file as binary/non-text: %1" + ).arg(path)); + return false; + } + + const auto title = info.fileName(); + auto result = ParseMarkdownForIv(bytes, ParseOptions{ title }); + if (!result.ok) { + const auto &error = result.error; + if (error.startsWith(u"cmark-"_q)) { + DEBUG_LOG(("Native Markdown IV: cmark parse failure (%1): %2" + ).arg(error + ).arg(path)); + } else { + DEBUG_LOG(("Native Markdown IV: parse failure (%1): %2" + ).arg(error + ).arg(path)); + } + return false; + } + if (!AcceptsPreview(result.document)) { + DEBUG_LOG(("Native Markdown IV: unsupported or empty document: %1" + ).arg(path)); + return false; + } + + OpenDocumentWindow(std::move(result.document), title); + DEBUG_LOG(("Native Markdown IV: opened as native Markdown IV: %1" + ).arg(path)); + return true; } } // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.h b/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.h index 12feb9829c..d25420c231 100644 --- a/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.h +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_controller.h @@ -1,18 +1,9 @@ #pragma once -#include "iv/markdown/iv_markdown_common.h" +#include namespace Iv::Markdown { -class Controller final { -public: - explicit Controller(OpenOptions options = {}); - - [[nodiscard]] const OpenOptions &options() const; - -private: - OpenOptions _options; - -}; +[[nodiscard]] bool TryOpenLocalFile(const QString &path); } // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_document.cpp b/Telegram/SourceFiles/iv/markdown/iv_markdown_document.cpp index 81a6499bf8..ef6df62b63 100644 --- a/Telegram/SourceFiles/iv/markdown/iv_markdown_document.cpp +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_document.cpp @@ -3,11 +3,272 @@ #include namespace Iv::Markdown { +namespace { + +[[nodiscard]] QString FromLatin1(const char *value) { + return QString::fromLatin1(value); +} + +[[nodiscard]] QString BoolString(bool value) { + return FromLatin1(value ? "true" : "false"); +} + +[[nodiscard]] QString NodeKindName(NodeKind kind) { + switch (kind) { + case NodeKind::Document: return FromLatin1("Document"); + case NodeKind::Paragraph: return FromLatin1("Paragraph"); + case NodeKind::Heading: return FromLatin1("Heading"); + case NodeKind::Text: return FromLatin1("Text"); + case NodeKind::Emphasis: return FromLatin1("Emphasis"); + case NodeKind::Strong: return FromLatin1("Strong"); + case NodeKind::Strike: return FromLatin1("Strike"); + case NodeKind::InlineCode: return FromLatin1("InlineCode"); + case NodeKind::CodeBlock: return FromLatin1("CodeBlock"); + case NodeKind::Link: return FromLatin1("Link"); + case NodeKind::List: return FromLatin1("List"); + case NodeKind::ListItem: return FromLatin1("ListItem"); + case NodeKind::Blockquote: return FromLatin1("Blockquote"); + case NodeKind::ThematicBreak: return FromLatin1("ThematicBreak"); + case NodeKind::Table: return FromLatin1("Table"); + case NodeKind::TableRow: return FromLatin1("TableRow"); + case NodeKind::TableCell: return FromLatin1("TableCell"); + case NodeKind::HtmlInline: return FromLatin1("HtmlInline"); + case NodeKind::HtmlBlock: return FromLatin1("HtmlBlock"); + case NodeKind::DisplayMath: return FromLatin1("DisplayMath"); + case NodeKind::InlineMath: return FromLatin1("InlineMath"); + case NodeKind::SoftBreak: return FromLatin1("SoftBreak"); + case NodeKind::LineBreak: return FromLatin1("LineBreak"); + case NodeKind::Unsupported: return FromLatin1("Unsupported"); + } + return FromLatin1("Unsupported"); +} + +[[nodiscard]] QString MathKindName(MathKind kind) { + switch (kind) { + case MathKind::Inline: return FromLatin1("Inline"); + case MathKind::Display: return FromLatin1("Display"); + } + return FromLatin1("Inline"); +} + +[[nodiscard]] QString ListKindName(ListKind kind) { + switch (kind) { + case ListKind::Bullet: return FromLatin1("Bullet"); + case ListKind::Ordered: return FromLatin1("Ordered"); + } + return FromLatin1("Bullet"); +} + +[[nodiscard]] QString ListDelimiterName(ListDelimiter delimiter) { + switch (delimiter) { + case ListDelimiter::None: return FromLatin1("None"); + case ListDelimiter::Period: return FromLatin1("Period"); + case ListDelimiter::Parenthesis: return FromLatin1("Parenthesis"); + } + return FromLatin1("None"); +} + +[[nodiscard]] QString TaskStateName(TaskState state) { + switch (state) { + case TaskState::None: return FromLatin1("None"); + case TaskState::Unchecked: return FromLatin1("Unchecked"); + case TaskState::Checked: return FromLatin1("Checked"); + } + return FromLatin1("None"); +} + +[[nodiscard]] QString TableAlignmentName(TableAlignment alignment) { + switch (alignment) { + case TableAlignment::None: return FromLatin1("None"); + case TableAlignment::Left: return FromLatin1("Left"); + case TableAlignment::Center: return FromLatin1("Center"); + case TableAlignment::Right: return FromLatin1("Right"); + } + return FromLatin1("None"); +} + +[[nodiscard]] QString RangeString(const SourceRange &range) { + if (!range.available) { + return FromLatin1("unavailable"); + } + return FromLatin1("%1:%2-%3:%4[%5,%6]").arg( + range.startLine + ).arg( + range.startColumn + ).arg( + range.endLine + ).arg( + range.endColumn + ).arg( + range.startOffset + ).arg( + range.endOffset); +} + +[[nodiscard]] QString EscapedValue(QString value) { + value.replace(FromLatin1("\\"), FromLatin1("\\\\")); + value.replace(FromLatin1("\r"), FromLatin1("\\r")); + value.replace(FromLatin1("\n"), FromLatin1("\\n")); + value.replace(FromLatin1("\t"), FromLatin1("\\t")); + return value; +} + +void AddRequiredStringAttribute( + QString *line, + const char *name, + const QString &value) { + line->append(FromLatin1(" ")); + line->append(FromLatin1(name)); + line->append(FromLatin1("=\"")); + line->append(EscapedValue(value)); + line->append(FromLatin1("\"")); +} + +void AddStringAttribute(QString *line, const char *name, const QString &value) { + if (value.isEmpty()) { + return; + } + AddRequiredStringAttribute(line, name, value); +} + +void AddIntAttribute(QString *line, const char *name, int value) { + line->append(FromLatin1(" ")); + line->append(FromLatin1(name)); + line->append(FromLatin1("=")); + line->append(QString::number(value)); +} + +void AddBoolAttribute(QString *line, const char *name, bool value) { + if (!value) { + return; + } + line->append(FromLatin1(" ")); + line->append(FromLatin1(name)); + line->append(FromLatin1("=")); + line->append(BoolString(value)); +} + +[[nodiscard]] QString TableAlignmentsString( + const std::vector &alignments) { + auto names = QStringList(); + for (const auto alignment : alignments) { + names.append(TableAlignmentName(alignment)); + } + return names.join(FromLatin1(",")); +} + +void DumpNode( + const MarkdownNode &node, + int depth, + QStringList *lines) { + auto line = FromLatin1("node"); + AddIntAttribute(&line, "depth", depth); + AddStringAttribute(&line, "kind", NodeKindName(node.kind)); + AddStringAttribute(&line, "range", RangeString(node.range)); + AddIntAttribute(&line, "children", static_cast(node.children.size())); + AddStringAttribute(&line, "text", node.text); + AddStringAttribute(&line, "url", node.url); + AddStringAttribute(&line, "title", node.title); + AddStringAttribute(&line, "info", node.info); + AddStringAttribute(&line, "raw", node.raw); + AddStringAttribute(&line, "unsupportedKind", node.unsupportedKind); + if (node.headingLevel != 0) { + AddIntAttribute(&line, "headingLevel", node.headingLevel); + } + if (node.listStart != 0) { + AddIntAttribute(&line, "listStart", node.listStart); + } + if (node.tableColumn != -1) { + AddIntAttribute(&line, "tableColumn", node.tableColumn); + } + if (node.formulaIndex != -1) { + AddIntAttribute(&line, "formulaIndex", node.formulaIndex); + } + if (node.kind == NodeKind::List) { + AddStringAttribute(&line, "listKind", ListKindName(node.listKind)); + AddStringAttribute( + &line, + "listDelimiter", + ListDelimiterName(node.listDelimiter)); + } + if (node.taskState != TaskState::None) { + AddStringAttribute(&line, "taskState", TaskStateName(node.taskState)); + } + AddBoolAttribute(&line, "tight", node.tight); + AddBoolAttribute(&line, "autolink", node.autolink); + AddBoolAttribute(&line, "tableHeader", node.tableHeader); + if (!node.tableAlignments.empty()) { + AddStringAttribute( + &line, + "tableAlignments", + TableAlignmentsString(node.tableAlignments)); + } + lines->append(line); + for (const auto &child : node.children) { + DumpNode(child, depth + 1, lines); + } +} + +} // namespace PreparedDocument EmptyDocument(QString sourceName) { auto document = PreparedDocument(); document.sourceName = std::move(sourceName); + document.document.kind = NodeKind::Document; return document; } +QString DumpForDebug(const PreparedDocument &document) { + auto lines = QStringList(); + lines.append(FromLatin1("sourceName=\"%1\"").arg( + EscapedValue(document.sourceName))); + lines.append(FromLatin1("title=\"%1\"").arg( + EscapedValue(document.title))); + lines.append(FromLatin1("sourceLength=%1").arg( + static_cast(document.sourceText.size()))); + lines.append(FromLatin1("empty=%1").arg(BoolString(document.empty))); + lines.append(FromLatin1("cmarkNodeCount=%1").arg( + document.stats.cmarkNodeCount)); + lines.append(FromLatin1("convertedNodeCount=%1").arg( + document.stats.convertedNodeCount)); + lines.append(FromLatin1("maxDepth=%1").arg( + document.stats.maxDepth)); + lines.append(FromLatin1("inlineFormulaCount=%1").arg( + document.stats.inlineFormulaCount)); + lines.append(FromLatin1("displayFormulaCount=%1").arg( + document.stats.displayFormulaCount)); + lines.append(FromLatin1("tablesSeen=%1").arg( + BoolString(document.stats.tablesSeen))); + lines.append(FromLatin1("taskListsSeen=%1").arg( + BoolString(document.stats.taskListsSeen))); + lines.append(FromLatin1("strikethroughSeen=%1").arg( + BoolString(document.stats.strikethroughSeen))); + lines.append(FromLatin1("autolinksSeen=%1").arg( + BoolString(document.stats.autolinksSeen))); + lines.append(FromLatin1("footnotesSeen=%1").arg( + BoolString(document.stats.footnotesSeen))); + lines.append(FromLatin1("warnings=%1").arg( + static_cast(document.warnings.size()))); + for (auto i = qsizetype(0); i != document.warnings.size(); ++i) { + lines.append(FromLatin1("warning index=%1 text=\"%2\"").arg( + i + 1 + ).arg( + EscapedValue(document.warnings[i]))); + } + lines.append(FromLatin1("nodes=preorder")); + DumpNode(document.document, 0, &lines); + lines.append(FromLatin1("formulas=%1").arg( + static_cast(document.formulas.size()))); + for (const auto &formula : document.formulas) { + auto line = FromLatin1("formula"); + AddIntAttribute(&line, "index", formula.index); + AddStringAttribute(&line, "kind", MathKindName(formula.kind)); + AddStringAttribute(&line, "range", RangeString(formula.range)); + AddRequiredStringAttribute(&line, "parent", formula.parentNodeKind); + AddRequiredStringAttribute(&line, "tex", formula.tex); + lines.append(line); + } + return lines.join(FromLatin1("\n")); +} + } // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_document.h b/Telegram/SourceFiles/iv/markdown/iv_markdown_document.h index 839efbff2e..ecca8fc024 100644 --- a/Telegram/SourceFiles/iv/markdown/iv_markdown_document.h +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_document.h @@ -2,11 +2,130 @@ #include "iv/markdown/iv_markdown_common.h" +#include + +#include + namespace Iv::Markdown { +enum class NodeKind { + Document, + Paragraph, + Heading, + Text, + Emphasis, + Strong, + Strike, + InlineCode, + CodeBlock, + Link, + List, + ListItem, + Blockquote, + ThematicBreak, + Table, + TableRow, + TableCell, + HtmlInline, + HtmlBlock, + DisplayMath, + InlineMath, + SoftBreak, + LineBreak, + Unsupported, +}; + +enum class MathKind { + Inline, + Display, +}; + +enum class ListKind { + Bullet, + Ordered, +}; + +enum class ListDelimiter { + None, + Period, + Parenthesis, +}; + +enum class TaskState { + None, + Unchecked, + Checked, +}; + +enum class TableAlignment { + None, + Left, + Center, + Right, +}; + +struct SourceRange { + bool available = false; + int startLine = 0; + int startColumn = 0; + int endLine = 0; + int endColumn = 0; + int startOffset = 0; + int endOffset = 0; +}; + +struct MarkdownNode { + NodeKind kind = NodeKind::Unsupported; + SourceRange range; + QString text; + QString url; + QString title; + QString info; + QString raw; + QString unsupportedKind; + std::vector children; + std::vector tableAlignments; + int headingLevel = 0; + int listStart = 0; + int tableColumn = -1; + int formulaIndex = -1; + ListKind listKind = ListKind::Bullet; + ListDelimiter listDelimiter = ListDelimiter::None; + TaskState taskState = TaskState::None; + bool tight = false; + bool autolink = false; + bool tableHeader = false; +}; + +struct MathFormula { + int index = 0; + MathKind kind = MathKind::Inline; + QString tex; + SourceRange range; + QString parentNodeKind; +}; + +struct ParseStats { + int cmarkNodeCount = 0; + int convertedNodeCount = 0; + int maxDepth = 0; + int inlineFormulaCount = 0; + int displayFormulaCount = 0; + bool tablesSeen = false; + bool taskListsSeen = false; + bool strikethroughSeen = false; + bool autolinksSeen = false; + bool footnotesSeen = false; +}; + struct PreparedDocument { QString sourceName; QString title; + QString sourceText; + MarkdownNode document; + std::vector formulas; + ParseStats stats; + QStringList warnings; bool empty = true; }; @@ -17,5 +136,6 @@ struct ParseResult { }; [[nodiscard]] PreparedDocument EmptyDocument(QString sourceName = QString()); +[[nodiscard]] QString DumpForDebug(const PreparedDocument &document); } // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_math.cpp b/Telegram/SourceFiles/iv/markdown/iv_markdown_math.cpp new file mode 100644 index 0000000000..87ecae6092 --- /dev/null +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_math.cpp @@ -0,0 +1,348 @@ +#include "iv/markdown/iv_markdown_math.h" + +#include + +#include +#include +#include +#include + +namespace Iv::Markdown { +namespace { + +[[nodiscard]] QString FromLatin1(const char *value) { + return QString::fromLatin1(value); +} + +[[nodiscard]] bool SetError(QString *error, const char *value) { + if (error) { + *error = FromLatin1(value); + } + return false; +} + +[[nodiscard]] bool IsMasked(const std::vector &mask, int offset) { + return (offset >= 0) + && (offset < static_cast(mask.size())) + && mask[offset]; +} + +[[nodiscard]] bool IsAsciiLetter(char ch) { + return ((ch >= 'a') && (ch <= 'z')) + || ((ch >= 'A') && (ch <= 'Z')); +} + +void OffsetToPosition( + const std::vector &lineStarts, + int offset, + int *line, + int *column) { + const auto clampedOffset = std::max(offset, 0); + auto resultLine = 1; + auto resultColumn = clampedOffset + 1; + if (!lineStarts.empty()) { + auto i = std::upper_bound( + lineStarts.begin(), + lineStarts.end(), + clampedOffset); + if (i != lineStarts.begin()) { + --i; + resultLine = static_cast(i - lineStarts.begin()) + 1; + resultColumn = clampedOffset - *i + 1; + } + } + if (line) { + *line = resultLine; + } + if (column) { + *column = resultColumn; + } +} + +[[nodiscard]] QString FindParentBlock( + const std::vector &blocks, + int line, + int column) { + auto best = FromLatin1("document"); + auto bestSpan = std::numeric_limits::max(); + for (const auto &block : blocks) { + const auto &range = block.range; + if (!range.available) { + continue; + } else if (line < range.startLine || line > range.endLine) { + continue; + } else if (line == range.startLine && column < range.startColumn) { + continue; + } else if (line == range.endLine && column > range.endColumn) { + continue; + } + const auto span = (range.endLine - range.startLine) * 1000 + + (range.endColumn - range.startColumn); + if (span < bestSpan) { + bestSpan = span; + best = block.nodeKind; + } + } + return best; +} + +[[nodiscard]] bool LooksLikeCurrency(const QByteArray &content) { + if (content.isEmpty()) { + return true; + } + auto hasDigit = false; + const auto size = static_cast(content.size()); + for (auto i = 0; i != size; ++i) { + const auto ch = content.at(i); + if (ch >= '0' && ch <= '9') { + hasDigit = true; + } else if (ch != '.' && ch != ',' && ch != ' ') { + return false; + } + } + return hasDigit; +} + +[[nodiscard]] bool LooksLikeProse(const QByteArray &content) { + auto alphaWords = 0; + auto longestRun = 0; + const auto size = static_cast(content.size()); + auto i = 0; + while (i < size) { + while (i < size && !IsAsciiLetter(content.at(i))) { + ++i; + } + const auto start = i; + while (i < size && IsAsciiLetter(content.at(i))) { + ++i; + } + const auto length = i - start; + if (length >= 2 && (start == 0 || content.at(start - 1) != '\\')) { + ++alphaWords; + longestRun = std::max(longestRun, length); + } + } + return alphaWords >= 2 || longestRun >= 5; +} + +[[nodiscard]] QByteArray StripBlockquoteMarkers(QByteArray content) { + auto cleaned = QByteArray(); + cleaned.reserve(content.size()); + const auto size = static_cast(content.size()); + auto position = 0; + while (position < size) { + auto endOfLine = position; + while (endOfLine < size && content.at(endOfLine) != '\n') { + ++endOfLine; + } + auto start = position; + if (position > 0) { + while (start < endOfLine) { + while (start < endOfLine + && (content.at(start) == ' ' || content.at(start) == '\t')) { + ++start; + } + if (start < endOfLine && content.at(start) == '>') { + ++start; + continue; + } + break; + } + if (start < endOfLine && content.at(start) == ' ') { + ++start; + } + } + cleaned.append(content.constData() + start, endOfLine - start); + if (endOfLine < size) { + cleaned.append('\n'); + } + position = endOfLine + 1; + } + return cleaned; +} + +[[nodiscard]] SourceRange RangeForOffsets( + const std::vector &lineStarts, + int sourceSize, + int startOffset, + int endOffset) { + auto startLine = 0; + auto startColumn = 0; + auto endLine = 0; + auto endColumn = 0; + OffsetToPosition(lineStarts, startOffset, &startLine, &startColumn); + OffsetToPosition( + lineStarts, + std::max(startOffset, endOffset - 1), + &endLine, + &endColumn); + return RangeFromLineColumns( + lineStarts, + sourceSize, + startLine, + startColumn, + endLine, + endColumn); +} + +[[nodiscard]] bool ExceedsFormulaBytes( + const QByteArray &content, + int maxFormulaBytes) { + return (maxFormulaBytes >= 0) + && (content.size() > maxFormulaBytes); +} + +[[nodiscard]] bool ExceedsFormulaCount( + const std::vector &formulas, + int maxFormulaCount) { + return (maxFormulaCount >= 0) + && (static_cast(formulas.size()) >= maxFormulaCount); +} + +} // namespace + +std::vector BuildLineStarts(const QByteArray &source) { + auto result = std::vector(); + result.push_back(0); + const auto size = static_cast(source.size()); + for (auto i = 0; i != size; ++i) { + if (source.at(i) == '\n') { + result.push_back(i + 1); + } + } + return result; +} + +SourceRange RangeFromLineColumns( + const std::vector &lineStarts, + int sourceSize, + int startLine, + int startColumn, + int endLine, + int endColumn) { + auto result = SourceRange(); + result.startLine = startLine; + result.startColumn = startColumn; + result.endLine = endLine; + result.endColumn = endColumn; + const auto linesCount = static_cast(lineStarts.size()); + if (startLine <= 0 + || endLine <= 0 + || startLine > linesCount + || endLine > linesCount) { + return result; + } + const auto maxOffset = std::max(sourceSize, 0); + const auto startOffset = lineStarts[startLine - 1] + + std::max(0, startColumn - 1); + const auto endOffset = lineStarts[endLine - 1] + + std::max(0, endColumn); + result.available = true; + result.startOffset = std::clamp(startOffset, 0, maxOffset); + result.endOffset = std::clamp(endOffset, 0, maxOffset); + return result; +} + +bool ExtractMathRegions( + const QByteArray &source, + const std::vector &mask, + const std::vector &lineStarts, + const std::vector &blocks, + int maxFormulaBytes, + int maxFormulaCount, + std::vector *out, + QString *error) { + if (!out) { + return SetError(error, "invalid-output"); + } + auto formulas = std::vector(); + const auto size = static_cast(source.size()); + auto i = 0; + while (i < size) { + if (IsMasked(mask, i) || source.at(i) != '$') { + ++i; + continue; + } + if (i > 0 && source.at(i - 1) == '\\') { + ++i; + continue; + } + const auto display = (i + 1 < size) + && (source.at(i + 1) == '$') + && !IsMasked(mask, i + 1); + const auto delimiterSize = display ? 2 : 1; + const auto contentStart = i + delimiterSize; + auto j = contentStart; + auto closing = -1; + while (j < size) { + if (IsMasked(mask, j)) { + ++j; + continue; + } + if (source.at(j) == '\\') { + j += 2; + continue; + } + if (!display && source.at(j) == '\n') { + break; + } + if (!display && source.at(j) == '$') { + closing = j; + break; + } + if (display + && source.at(j) == '$' + && (j + 1 < size) + && source.at(j + 1) == '$' + && !IsMasked(mask, j + 1)) { + closing = j; + break; + } + ++j; + } + if (closing < 0) { + ++i; + continue; + } + auto content = source.mid(contentStart, closing - contentStart); + if (!display && LooksLikeCurrency(content)) { + ++i; + continue; + } + if (!display && LooksLikeProse(content)) { + ++i; + continue; + } + if (display) { + content = StripBlockquoteMarkers(std::move(content)); + } + if (ExceedsFormulaBytes(content, maxFormulaBytes)) { + return SetError(error, "formula-too-large"); + } + if (ExceedsFormulaCount(formulas, maxFormulaCount)) { + return SetError(error, "too-many-formulas"); + } + auto startLine = 0; + auto startColumn = 0; + OffsetToPosition(lineStarts, i, &startLine, &startColumn); + auto formula = MathFormula(); + formula.index = static_cast(formulas.size()) + 1; + formula.kind = display ? MathKind::Display : MathKind::Inline; + formula.tex = QString::fromUtf8(content.constData(), content.size()); + formula.range = RangeForOffsets( + lineStarts, + size, + i, + closing + delimiterSize); + formula.parentNodeKind = FindParentBlock(blocks, startLine, startColumn); + formulas.push_back(std::move(formula)); + i = closing + delimiterSize; + } + *out = std::move(formulas); + if (error) { + error->clear(); + } + return true; +} + +} // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_math.h b/Telegram/SourceFiles/iv/markdown/iv_markdown_math.h new file mode 100644 index 0000000000..c10e7bdc6c --- /dev/null +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_math.h @@ -0,0 +1,34 @@ +#pragma once + +#include "iv/markdown/iv_markdown_document.h" + +#include + +#include + +namespace Iv::Markdown { + +struct MathScanBlock { + SourceRange range; + QString nodeKind; +}; + +[[nodiscard]] std::vector BuildLineStarts(const QByteArray &source); +[[nodiscard]] SourceRange RangeFromLineColumns( + const std::vector &lineStarts, + int sourceSize, + int startLine, + int startColumn, + int endLine, + int endColumn); +[[nodiscard]] bool ExtractMathRegions( + const QByteArray &source, + const std::vector &mask, + const std::vector &lineStarts, + const std::vector &blocks, + int maxFormulaBytes, + int maxFormulaCount, + std::vector *out, + QString *error); + +} // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_parse.cpp b/Telegram/SourceFiles/iv/markdown/iv_markdown_parse.cpp index d811f98904..67f5a24443 100644 --- a/Telegram/SourceFiles/iv/markdown/iv_markdown_parse.cpp +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_parse.cpp @@ -1,13 +1,1009 @@ #include "iv/markdown/iv_markdown_parse.h" +#include "iv/markdown/iv_markdown_math.h" + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include #include +#include namespace Iv::Markdown { +namespace { -ParseResult ParseMarkdownForIv(const QByteArray &, ParseOptions options) { +constexpr auto kMaxSourceBytes = 4 * 1024 * 1024; +constexpr auto kMaxCmarkNodes = 100000; +constexpr auto kMaxNesting = 128; +constexpr auto kMaxFormulaBytes = 64 * 1024; +constexpr auto kMaxFormulaCount = 10000; + +struct ParserDeleter { + void operator()(cmark_parser *parser) const; +}; + +struct NodeDeleter { + void operator()(cmark_node *node) const; +}; + +using ParserPointer = std::unique_ptr; +using NodePointer = std::unique_ptr; + +struct ParserState { + const QByteArray &normalizedSource; + const std::vector &lineStarts; + std::vector *mask = nullptr; + std::vector *scanBlocks = nullptr; + ParseStats *stats = nullptr; + QStringList *warnings = nullptr; + QString error; + bool failed = false; +}; + +void ParserDeleter::operator()(cmark_parser *parser) const { + if (parser) { + cmark_parser_free(parser); + } +} + +void NodeDeleter::operator()(cmark_node *node) const { + if (node) { + cmark_node_free(node); + } +} + +[[nodiscard]] QString FromLatin1(const char *value) { + return QString::fromLatin1(value); +} + +[[nodiscard]] QString ExtensionError(const char *prefix, const char *name) { + return FromLatin1("%1-%2").arg( + FromLatin1(prefix), + FromLatin1(name)); +} + +[[nodiscard]] unsigned char ByteAt(const QByteArray &source, int index) { + return static_cast(source.at(index)); +} + +template +[[nodiscard]] bool HasPrefix( + const QByteArray &source, + const std::array &prefix) { + if (source.size() < static_cast(Size)) { + return false; + } + for (auto i = std::size_t(0); i != Size; ++i) { + if (ByteAt(source, static_cast(i)) != prefix[i]) { + return false; + } + } + return true; +} + +[[nodiscard]] ParseResult Failure(QString sourceName, QString error) { auto result = ParseResult(); - result.document = EmptyDocument(std::move(options.sourceName)); + result.document = EmptyDocument(std::move(sourceName)); + result.error = std::move(error); + result.ok = false; return result; } +[[nodiscard]] bool HasUtf8Bom(const QByteArray &source) { + constexpr auto kUtf8Bom = std::array{ + 0xEF, + 0xBB, + 0xBF, + }; + return HasPrefix(source, kUtf8Bom); +} + +[[nodiscard]] bool HasUnsupportedUnicodeBom(const QByteArray &source) { + constexpr auto kUtf32Boms = std::array{ + std::array{ 0x00, 0x00, 0xFE, 0xFF }, + std::array{ 0xFF, 0xFE, 0x00, 0x00 }, + }; + for (const auto &bom : kUtf32Boms) { + if (HasPrefix(source, bom)) { + return true; + } + } + constexpr auto kUtf16Boms = std::array{ + std::array{ 0xFE, 0xFF }, + std::array{ 0xFF, 0xFE }, + }; + for (const auto &bom : kUtf16Boms) { + if (HasPrefix(source, bom)) { + return true; + } + } + return false; +} + +[[nodiscard]] QByteArray StripUtf8Bom(QByteArray source) { + if (HasUtf8Bom(source)) { + source.remove(0, 3); + } + return source; +} + +[[nodiscard]] bool IsAllowedControl(unsigned char byte) { + return byte == '\t' || byte == '\n' || byte == '\r'; +} + +[[nodiscard]] bool LooksBinary(const QByteArray &source) { + const auto size = static_cast(source.size()); + if (size == 0) { + return false; + } + auto controlBytes = 0; + for (auto i = 0; i != size; ++i) { + const auto byte = ByteAt(source, i); + if (byte == 0) { + return true; + } else if ((byte < 0x20 || byte == 0x7F) + && !IsAllowedControl(byte)) { + ++controlBytes; + } + } + return (controlBytes * 10) > size; +} + +[[nodiscard]] bool IsUtf8Continuation(unsigned char byte) { + return (byte & 0xC0) == 0x80; +} + +[[nodiscard]] bool IsValidCodepoint(int codepoint, int minimum) { + return codepoint >= minimum + && codepoint <= 0x10FFFF + && (codepoint < 0xD800 || codepoint > 0xDFFF); +} + +[[nodiscard]] bool IsValidUtf8(const QByteArray &source) { + const auto size = static_cast(source.size()); + for (auto i = 0; i != size; ++i) { + const auto byte = ByteAt(source, i); + if (byte <= 0x7F) { + continue; + } + auto extraBytes = 0; + auto codepoint = 0; + auto minimum = 0; + if (byte >= 0xC2 && byte <= 0xDF) { + extraBytes = 1; + codepoint = byte & 0x1F; + minimum = 0x80; + } else if (byte >= 0xE0 && byte <= 0xEF) { + extraBytes = 2; + codepoint = byte & 0x0F; + minimum = 0x800; + } else if (byte >= 0xF0 && byte <= 0xF4) { + extraBytes = 3; + codepoint = byte & 0x07; + minimum = 0x10000; + } else { + return false; + } + if (i + extraBytes >= size) { + return false; + } + for (auto j = 1; j <= extraBytes; ++j) { + const auto continuation = ByteAt(source, i + j); + if (!IsUtf8Continuation(continuation)) { + return false; + } + codepoint = (codepoint << 6) | (continuation & 0x3F); + } + if (!IsValidCodepoint(codepoint, minimum)) { + return false; + } + i += extraBytes; + } + return true; +} + +[[nodiscard]] bool ValidateSource( + const QByteArray &source, + QByteArray *normalized, + QString *decoded, + QString *error) { + const auto fail = [=](const char *value) { + if (error) { + *error = FromLatin1(value); + } + return false; + }; + if (source.size() > kMaxSourceBytes) { + return fail("source-too-large"); + } + if (HasUnsupportedUnicodeBom(source)) { + return fail("source-unsupported-bom"); + } + auto normalizedSource = StripUtf8Bom(source); + if (LooksBinary(normalizedSource)) { + return fail("source-binary"); + } + if (!IsValidUtf8(normalizedSource)) { + return fail("source-invalid-utf8"); + } + if (decoded) { + *decoded = QString::fromUtf8( + normalizedSource.constData(), + normalizedSource.size()); + } + if (normalized) { + *normalized = std::move(normalizedSource); + } + if (error) { + error->clear(); + } + return true; +} + +void EnsureCmarkExtensionsRegistered() { + static std::once_flag once; + std::call_once(once, [] { + cmark_gfm_core_extensions_ensure_registered(); + }); +} + +[[nodiscard]] bool AttachExtensions(cmark_parser *parser, QString *error) { + if (!parser) { + if (error) { + *error = FromLatin1("cmark-parser-failed"); + } + return false; + } + EnsureCmarkExtensionsRegistered(); + constexpr auto kExtensions = std::array{ + "table", + "strikethrough", + "autolink", + "tagfilter", + "tasklist", + }; + for (const auto name : kExtensions) { + const auto extension = cmark_find_syntax_extension(name); + if (!extension) { + if (error) { + *error = ExtensionError("cmark-extension-missing", name); + } + return false; + } + if (!cmark_parser_attach_syntax_extension(parser, extension)) { + if (error) { + *error = ExtensionError("cmark-extension-attach-failed", name); + } + return false; + } + } + if (error) { + error->clear(); + } + return true; +} + +[[nodiscard]] QString FromCmarkString(const char *value) { + return value ? QString::fromUtf8(value) : QString(); +} + +[[nodiscard]] QString RawTypeString(cmark_node *node) { + return node ? FromCmarkString(cmark_node_get_type_string(node)) : QString(); +} + +[[nodiscard]] bool IsCoreNodeType(cmark_node_type type) { + constexpr auto kCoreTypes = std::array{ + CMARK_NODE_DOCUMENT, + CMARK_NODE_BLOCK_QUOTE, + CMARK_NODE_LIST, + CMARK_NODE_ITEM, + CMARK_NODE_CODE_BLOCK, + CMARK_NODE_HTML_BLOCK, + CMARK_NODE_CUSTOM_BLOCK, + CMARK_NODE_PARAGRAPH, + CMARK_NODE_HEADING, + CMARK_NODE_THEMATIC_BREAK, + CMARK_NODE_FOOTNOTE_DEFINITION, + CMARK_NODE_TEXT, + CMARK_NODE_SOFTBREAK, + CMARK_NODE_LINEBREAK, + CMARK_NODE_CODE, + CMARK_NODE_HTML_INLINE, + CMARK_NODE_CUSTOM_INLINE, + CMARK_NODE_EMPH, + CMARK_NODE_STRONG, + CMARK_NODE_LINK, + CMARK_NODE_IMAGE, + CMARK_NODE_FOOTNOTE_REFERENCE, + }; + return std::find(kCoreTypes.begin(), kCoreTypes.end(), type) + != kCoreTypes.end(); +} + +[[nodiscard]] QString CmarkKind(cmark_node *node) { + if (!node) { + return FromLatin1("unknown"); + } + const auto type = cmark_node_get_type(node); + const auto raw = RawTypeString(node); + if (!IsCoreNodeType(type)) { + return raw.isEmpty() ? FromLatin1("unknown") : raw; + } + switch (type) { + case CMARK_NODE_DOCUMENT: return FromLatin1("document"); + case CMARK_NODE_BLOCK_QUOTE: return FromLatin1("block_quote"); + case CMARK_NODE_LIST: return FromLatin1("list"); + case CMARK_NODE_ITEM: return FromLatin1("item"); + case CMARK_NODE_CODE_BLOCK: return FromLatin1("code_block"); + case CMARK_NODE_HTML_BLOCK: return FromLatin1("html_block"); + case CMARK_NODE_CUSTOM_BLOCK: return FromLatin1("custom_block"); + case CMARK_NODE_PARAGRAPH: return FromLatin1("paragraph"); + case CMARK_NODE_HEADING: return FromLatin1("heading"); + case CMARK_NODE_THEMATIC_BREAK: return FromLatin1("thematic_break"); + case CMARK_NODE_FOOTNOTE_DEFINITION: + return FromLatin1("footnote_definition"); + case CMARK_NODE_TEXT: return FromLatin1("text"); + case CMARK_NODE_SOFTBREAK: return FromLatin1("softbreak"); + case CMARK_NODE_LINEBREAK: return FromLatin1("linebreak"); + case CMARK_NODE_CODE: return FromLatin1("code"); + case CMARK_NODE_HTML_INLINE: return FromLatin1("html_inline"); + case CMARK_NODE_CUSTOM_INLINE: return FromLatin1("custom_inline"); + case CMARK_NODE_EMPH: return FromLatin1("emph"); + case CMARK_NODE_STRONG: return FromLatin1("strong"); + case CMARK_NODE_LINK: return FromLatin1("link"); + case CMARK_NODE_IMAGE: return FromLatin1("image"); + case CMARK_NODE_FOOTNOTE_REFERENCE: + return FromLatin1("footnote_reference"); + case CMARK_NODE_NONE: return FromLatin1("none"); + } + return raw.isEmpty() ? FromLatin1("unknown") : raw; +} + +[[nodiscard]] SourceRange RangeFromCmarkLines( + const std::vector &lineStarts, + int sourceSize, + int startLine, + int startColumn, + int endLine, + int endColumn) { + auto result = SourceRange(); + result.startLine = startLine; + result.startColumn = startColumn; + result.endLine = endLine; + result.endColumn = endColumn; + const auto linesCount = static_cast(lineStarts.size()); + if (startLine <= 0 + || endLine <= 0 + || startLine > linesCount + || endLine > linesCount) { + return result; + } + const auto maxOffset = std::max(sourceSize, 0); + const auto startOffset = lineStarts[startLine - 1] + + std::max(0, startColumn - 1); + const auto endOffset = lineStarts[endLine - 1] + + std::max(0, endColumn); + result.available = true; + result.startOffset = std::clamp(startOffset, 0, maxOffset); + result.endOffset = std::clamp(endOffset, 0, maxOffset); + return result; +} + +[[nodiscard]] SourceRange NodeRange( + cmark_node *node, + const std::vector &lineStarts, + int sourceSize) { + return node + ? RangeFromCmarkLines( + lineStarts, + sourceSize, + cmark_node_get_start_line(node), + cmark_node_get_start_column(node), + cmark_node_get_end_line(node), + cmark_node_get_end_column(node)) + : SourceRange(); +} + +[[nodiscard, maybe_unused]] QString SourceSlice( + const QByteArray &source, + const SourceRange &range) { + if (!range.available) { + return QString(); + } + const auto sourceSize = static_cast(source.size()); + const auto start = std::clamp(range.startOffset, 0, sourceSize); + const auto end = std::clamp(range.endOffset, start, sourceSize); + const auto slice = source.mid(start, end - start); + return QString::fromUtf8(slice.constData(), slice.size()); +} + +template +[[nodiscard]] bool IsAnyKind( + const QString &kind, + const std::array &values) { + for (const auto value : values) { + if (kind == FromLatin1(value)) { + return true; + } + } + return false; +} + +[[nodiscard]] bool HasCmarkSourceRange(cmark_node *node) { + return node + && cmark_node_get_start_line(node) > 0 + && cmark_node_get_end_line(node) > 0; +} + +[[nodiscard]] bool IsBlockNodeType(cmark_node_type type) { + return (type & CMARK_NODE_TYPE_MASK) == CMARK_NODE_TYPE_BLOCK; +} + +void MarkMaskRange(std::vector *mask, const SourceRange &range) { + if (!mask || !range.available) { + return; + } + const auto size = static_cast(mask->size()); + const auto start = std::clamp(range.startOffset, 0, size); + const auto end = std::clamp(range.endOffset, start, size); + for (auto i = start; i != end; ++i) { + (*mask)[i] = true; + } +} + +[[nodiscard]] bool IsMathMaskedNode(cmark_node *node) { + if (!node) { + return false; + } + constexpr auto kMaskedTypes = std::array{ + CMARK_NODE_CODE, + CMARK_NODE_CODE_BLOCK, + CMARK_NODE_HTML_INLINE, + CMARK_NODE_HTML_BLOCK, + }; + const auto type = cmark_node_get_type(node); + return std::find(kMaskedTypes.begin(), kMaskedTypes.end(), type) + != kMaskedTypes.end(); +} + +[[nodiscard]] bool IsTaskListItem(cmark_node *node) { + if (!node + || cmark_node_get_type(node) != CMARK_NODE_ITEM + || RawTypeString(node) != FromLatin1("tasklist")) { + return false; + } + (void)cmark_gfm_extensions_get_tasklist_item_checked(node); + return true; +} + +[[nodiscard]] bool IsBlockForMathParent(cmark_node *node) { + if (!node) { + return false; + } + const auto kind = CmarkKind(node); + constexpr auto kBlockKinds = std::array{ + "paragraph", + "heading", + "list", + "item", + "block_quote", + "code_block", + "html_block", + "thematic_break", + "table", + "table_header", + "table_row", + "table_cell", + "footnote_definition", + }; + if (IsAnyKind(kind, kBlockKinds)) { + return true; + } + const auto type = cmark_node_get_type(node); + return !IsCoreNodeType(type) + && IsBlockNodeType(type) + && HasCmarkSourceRange(node); +} + +void RecordCapabilities(cmark_node *node, ParserState *state) { + if (!node || !state || !state->stats) { + return; + } + const auto kind = CmarkKind(node); + constexpr auto kTableKinds = std::array{ + "table", + "table_header", + "table_row", + "table_cell", + }; + if (IsAnyKind(kind, kTableKinds)) { + state->stats->tablesSeen = true; + } + if (IsTaskListItem(node)) { + state->stats->taskListsSeen = true; + } + if (kind == FromLatin1("strikethrough")) { + state->stats->strikethroughSeen = true; + } else if (kind == FromLatin1("footnote_reference") + || kind == FromLatin1("footnote_definition")) { + state->stats->footnotesSeen = true; + } +} + +[[nodiscard]] bool FailScanMetadata(ParserState *state, const char *error) { + if (state) { + state->error = FromLatin1(error); + state->failed = true; + } + return false; +} + +[[nodiscard]] bool CollectScanMetadata( + cmark_node *node, + ParserState *state, + int depth) { + if (!node || !state || state->failed) { + return false; + } + if (state->stats) { + state->stats->maxDepth = std::max(state->stats->maxDepth, depth); + } + if (depth > kMaxNesting) { + return FailScanMetadata(state, "cmark-nesting-too-deep"); + } + if (state->stats) { + ++state->stats->cmarkNodeCount; + if (state->stats->cmarkNodeCount > kMaxCmarkNodes) { + return FailScanMetadata(state, "too-many-cmark-nodes"); + } + } + RecordCapabilities(node, state); + const auto range = NodeRange( + node, + state->lineStarts, + static_cast(state->normalizedSource.size())); + if (IsBlockForMathParent(node) && state->scanBlocks) { + state->scanBlocks->push_back(MathScanBlock{ range, CmarkKind(node) }); + } + if (IsMathMaskedNode(node)) { + MarkMaskRange(state->mask, range); + } + for (auto child = cmark_node_first_child(node); child;) { + const auto next = cmark_node_next(child); + if (!CollectScanMetadata(child, state, depth + 1)) { + return false; + } + child = next; + } + return true; +} + +[[nodiscard]] NodeKind ExtensionNodeKind(const QString &raw) { + struct Entry { + const char *name = nullptr; + NodeKind kind = NodeKind::Unsupported; + }; + constexpr auto kEntries = std::array{ + Entry{ "strikethrough", NodeKind::Strike }, + Entry{ "table", NodeKind::Table }, + Entry{ "table_header", NodeKind::TableRow }, + Entry{ "table_row", NodeKind::TableRow }, + Entry{ "table_cell", NodeKind::TableCell }, + }; + for (const auto &entry : kEntries) { + if (raw == FromLatin1(entry.name)) { + return entry.kind; + } + } + return NodeKind::Unsupported; +} + +[[nodiscard]] NodeKind NodeKindFor(cmark_node *node) { + if (!node) { + return NodeKind::Unsupported; + } + const auto type = cmark_node_get_type(node); + switch (type) { + case CMARK_NODE_DOCUMENT: return NodeKind::Document; + case CMARK_NODE_BLOCK_QUOTE: return NodeKind::Blockquote; + case CMARK_NODE_LIST: return NodeKind::List; + case CMARK_NODE_ITEM: return NodeKind::ListItem; + case CMARK_NODE_CODE_BLOCK: return NodeKind::CodeBlock; + case CMARK_NODE_HTML_BLOCK: return NodeKind::HtmlBlock; + case CMARK_NODE_PARAGRAPH: return NodeKind::Paragraph; + case CMARK_NODE_HEADING: return NodeKind::Heading; + case CMARK_NODE_THEMATIC_BREAK: return NodeKind::ThematicBreak; + case CMARK_NODE_TEXT: return NodeKind::Text; + case CMARK_NODE_SOFTBREAK: return NodeKind::SoftBreak; + case CMARK_NODE_LINEBREAK: return NodeKind::LineBreak; + case CMARK_NODE_CODE: return NodeKind::InlineCode; + case CMARK_NODE_HTML_INLINE: return NodeKind::HtmlInline; + case CMARK_NODE_EMPH: return NodeKind::Emphasis; + case CMARK_NODE_STRONG: return NodeKind::Strong; + case CMARK_NODE_LINK: return NodeKind::Link; + default: break; + } + return ExtensionNodeKind(RawTypeString(node)); +} + +[[nodiscard]] ListKind ListKindFor(cmark_node *node) { + return (node && cmark_node_get_list_type(node) == CMARK_ORDERED_LIST) + ? ListKind::Ordered + : ListKind::Bullet; +} + +[[nodiscard]] ListDelimiter ListDelimiterFor(cmark_node *node) { + if (!node) { + return ListDelimiter::None; + } + switch (cmark_node_get_list_delim(node)) { + case CMARK_PERIOD_DELIM: return ListDelimiter::Period; + case CMARK_PAREN_DELIM: return ListDelimiter::Parenthesis; + case CMARK_NO_DELIM: return ListDelimiter::None; + } + return ListDelimiter::None; +} + +[[nodiscard]] TaskState TaskStateFor(cmark_node *node) { + if (!IsTaskListItem(node)) { + return TaskState::None; + } + return cmark_gfm_extensions_get_tasklist_item_checked(node) + ? TaskState::Checked + : TaskState::Unchecked; +} + +[[nodiscard]] TableAlignment TableAlignmentFor(uint8_t value) { + switch (value) { + case 'l': return TableAlignment::Left; + case 'c': return TableAlignment::Center; + case 'r': return TableAlignment::Right; + default: return TableAlignment::None; + } +} + +[[nodiscard]] std::vector TableAlignmentsFor(cmark_node *node) { + auto result = std::vector(); + if (!node) { + return result; + } + const auto columns = cmark_gfm_extensions_get_table_columns(node); + if (!columns) { + return result; + } + result.reserve(columns); + const auto alignments = cmark_gfm_extensions_get_table_alignments(node); + for (auto i = uint16_t(0); i != columns; ++i) { + result.push_back( + alignments + ? TableAlignmentFor(alignments[i]) + : TableAlignment::None); + } + return result; +} + +[[nodiscard]] bool TableRowIsHeader(cmark_node *node) { + return node && cmark_gfm_extensions_get_table_row_is_header(node) != 0; +} + +[[nodiscard]] int TableCellColumn(cmark_node *node) { + if (!node) { + return -1; + } + auto result = 0; + for (auto previous = cmark_node_previous(node); previous;) { + if (NodeKindFor(previous) == NodeKind::TableCell) { + ++result; + } + previous = cmark_node_previous(previous); + } + return result; +} + +[[nodiscard]] QString PlainText(const MarkdownNode &node) { + auto result = node.text; + for (const auto &child : node.children) { + result.append(PlainText(child)); + } + return result; +} + +[[nodiscard]] bool LinkLooksAutolink(const MarkdownNode &node) { + const auto text = PlainText(node); + if (text.isEmpty() || node.url.isEmpty()) { + return false; + } + if (text == node.url) { + return true; + } + const auto mailto = FromLatin1("mailto:"); + return node.url.startsWith(mailto, Qt::CaseInsensitive) + && text == node.url.mid(mailto.size()); +} + +void FillNodeAttributes(cmark_node *node, MarkdownNode *out) { + switch (out->kind) { + case NodeKind::Text: + case NodeKind::InlineCode: + out->text = FromCmarkString(cmark_node_get_literal(node)); + break; + case NodeKind::CodeBlock: + out->text = FromCmarkString(cmark_node_get_literal(node)); + out->info = FromCmarkString(cmark_node_get_fence_info(node)); + break; + case NodeKind::HtmlBlock: + case NodeKind::HtmlInline: + out->raw = FromCmarkString(cmark_node_get_literal(node)); + break; + case NodeKind::SoftBreak: + case NodeKind::LineBreak: + out->text = FromLatin1("\n"); + break; + case NodeKind::Heading: + out->headingLevel = cmark_node_get_heading_level(node); + break; + case NodeKind::List: + out->listKind = ListKindFor(node); + out->listDelimiter = ListDelimiterFor(node); + out->listStart = cmark_node_get_list_start(node); + out->tight = cmark_node_get_list_tight(node) != 0; + break; + case NodeKind::ListItem: + out->taskState = TaskStateFor(node); + break; + case NodeKind::Table: + out->tableAlignments = TableAlignmentsFor(node); + break; + case NodeKind::TableRow: + out->tableHeader = TableRowIsHeader(node); + break; + case NodeKind::TableCell: + out->tableColumn = TableCellColumn(node); + break; + case NodeKind::Link: + out->url = FromCmarkString(cmark_node_get_url(node)); + out->title = FromCmarkString(cmark_node_get_title(node)); + break; + default: + break; + } +} + +[[nodiscard]] bool ConvertNode( + cmark_node *node, + ParserState *state, + int depth, + MarkdownNode *out) { + if (!node || !state || !out || state->failed) { + return false; + } + if (depth > kMaxNesting) { + return FailScanMetadata(state, "cmark-nesting-too-deep"); + } + out->kind = NodeKindFor(node); + out->range = NodeRange( + node, + state->lineStarts, + static_cast(state->normalizedSource.size())); + if (out->kind == NodeKind::Unsupported) { + out->unsupportedKind = CmarkKind(node); + out->raw = SourceSlice(state->normalizedSource, out->range); + } + FillNodeAttributes(node, out); + for (auto child = cmark_node_first_child(node); child;) { + const auto next = cmark_node_next(child); + auto converted = MarkdownNode(); + if (!ConvertNode(child, state, depth + 1, &converted)) { + return false; + } + out->children.push_back(std::move(converted)); + child = next; + } + if (out->kind == NodeKind::Link) { + out->autolink = LinkLooksAutolink(*out); + if (out->autolink && state->stats) { + state->stats->autolinksSeen = true; + } + } + if (state->stats) { + ++state->stats->convertedNodeCount; + } + return true; +} + +[[nodiscard]] MarkdownNode DisplayMathNode( + const MathFormula &formula, + int vectorIndex) { + auto result = MarkdownNode(); + result.kind = NodeKind::DisplayMath; + result.range = formula.range; + result.text = formula.tex; + result.formulaIndex = vectorIndex; + return result; +} + +[[nodiscard]] int RangeStartOffset(const SourceRange &range) { + return range.available + ? range.startOffset + : std::numeric_limits::max(); +} + +[[nodiscard]] std::vector DisplayFormulaIndexes( + const std::vector &formulas) { + auto result = std::vector(); + const auto count = static_cast(formulas.size()); + for (auto i = 0; i != count; ++i) { + if (formulas[i].kind == MathKind::Display) { + result.push_back(i); + } + } + std::sort( + result.begin(), + result.end(), + [&](int left, int right) { + const auto leftOffset = RangeStartOffset(formulas[left].range); + const auto rightOffset = RangeStartOffset(formulas[right].range); + return (leftOffset != rightOffset) + ? (leftOffset < rightOffset) + : (left < right); + }); + return result; +} + +void InsertDisplayMathBlocks(PreparedDocument *document) { + if (!document) { + return; + } + const auto displayIndexes = DisplayFormulaIndexes(document->formulas); + if (displayIndexes.empty()) { + return; + } + auto originalChildren = std::move(document->document.children); + auto children = std::vector(); + children.reserve(originalChildren.size() + displayIndexes.size()); + auto displayIndex = std::size_t(0); + const auto appendDisplayBefore = [&](int offset) { + while (displayIndex != displayIndexes.size()) { + const auto formulaIndex = displayIndexes[displayIndex]; + const auto formulaOffset = RangeStartOffset( + document->formulas[formulaIndex].range); + if (formulaOffset > offset) { + break; + } + children.push_back(DisplayMathNode( + document->formulas[formulaIndex], + formulaIndex)); + ++displayIndex; + } + }; + for (auto &child : originalChildren) { + appendDisplayBefore(RangeStartOffset(child.range)); + children.push_back(std::move(child)); + } + appendDisplayBefore(std::numeric_limits::max()); + const auto displayCount = static_cast(displayIndexes.size()); + document->stats.convertedNodeCount += displayCount; + document->document.children = std::move(children); +} + +[[nodiscard]] QString FirstHeadingTitle(const MarkdownNode &node) { + if (node.kind == NodeKind::Heading) { + return PlainText(node).trimmed(); + } + for (const auto &child : node.children) { + const auto result = FirstHeadingTitle(child); + if (!result.isEmpty()) { + return result; + } + } + return QString(); +} + +void FillFormulaStats(PreparedDocument *document) { + if (!document) { + return; + } + document->stats.inlineFormulaCount = 0; + document->stats.displayFormulaCount = 0; + for (const auto &formula : document->formulas) { + switch (formula.kind) { + case MathKind::Inline: + ++document->stats.inlineFormulaCount; + break; + case MathKind::Display: + ++document->stats.displayFormulaCount; + break; + } + } +} + +} // namespace + +ParseResult ParseMarkdownForIv(const QByteArray &source, ParseOptions options) { + auto normalized = QByteArray(); + auto decoded = QString(); + auto error = QString(); + if (!ValidateSource(source, &normalized, &decoded, &error)) { + return Failure(std::move(options.sourceName), std::move(error)); + } + const auto lineStarts = BuildLineStarts(normalized); + auto mask = std::vector(normalized.size(), false); + const auto parserOptions = CMARK_OPT_DEFAULT + | CMARK_OPT_SOURCEPOS + | CMARK_OPT_FOOTNOTES + | CMARK_OPT_STRIKETHROUGH_DOUBLE_TILDE; + auto parser = ParserPointer(cmark_parser_new(parserOptions)); + if (!parser) { + return Failure( + std::move(options.sourceName), + FromLatin1("cmark-parser-failed")); + } + if (!AttachExtensions(parser.get(), &error)) { + return Failure(std::move(options.sourceName), std::move(error)); + } + cmark_parser_feed( + parser.get(), + normalized.constData(), + static_cast(normalized.size())); + auto root = NodePointer(cmark_parser_finish(parser.get())); + if (!root) { + return Failure( + std::move(options.sourceName), + FromLatin1("cmark-parser-failed")); + } + auto document = EmptyDocument(std::move(options.sourceName)); + document.sourceText = std::move(decoded); + auto scanBlocks = std::vector(); + auto state = ParserState{ + normalized, + lineStarts, + &mask, + &scanBlocks, + &document.stats, + &document.warnings, + }; + if (!CollectScanMetadata(root.get(), &state, 0)) { + return Failure(std::move(document.sourceName), std::move(state.error)); + } + if (!ExtractMathRegions( + normalized, + mask, + lineStarts, + scanBlocks, + kMaxFormulaBytes, + kMaxFormulaCount, + &document.formulas, + &error)) { + return Failure(std::move(document.sourceName), std::move(error)); + } + if (!ConvertNode( + root.get(), + &state, + 0, + &document.document)) { + return Failure( + std::move(document.sourceName), + state.error.isEmpty() + ? FromLatin1("cmark-conversion-failed") + : std::move(state.error)); + } + FillFormulaStats(&document); + document.title = FirstHeadingTitle(document.document); + document.empty = document.document.children.empty() + && document.formulas.empty(); + InsertDisplayMathBlocks(&document); + document.empty = document.document.children.empty() + && document.formulas.empty(); + return ParseResult{ std::move(document), QString(), true }; +} + } // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/iv/markdown/iv_markdown_view.cpp b/Telegram/SourceFiles/iv/markdown/iv_markdown_view.cpp index b3a73303df..a558b16e86 100644 --- a/Telegram/SourceFiles/iv/markdown/iv_markdown_view.cpp +++ b/Telegram/SourceFiles/iv/markdown/iv_markdown_view.cpp @@ -1,13 +1,88 @@ #include "iv/markdown/iv_markdown_view.h" +#include "ui/widgets/labels.h" +#include "ui/widgets/scroll_area.h" +#include "ui/wrap/padding_wrap.h" #include "ui/rp_widget.h" +#include "styles/style_boxes.h" +#include "styles/style_layers.h" + +#include + namespace Iv::Markdown { +namespace { + +constexpr auto kDiagnosticDumpLineCount = 20; + +[[nodiscard]] QString BuildDiagnosticText( + const PreparedDocument &document, + const OpenOptions &options) { + auto lines = QStringList(); + const auto sourceName = options.sourceName.isEmpty() + ? document.sourceName + : options.sourceName; + lines.append(QStringLiteral("Source: %1").arg(sourceName)); + if (!document.title.isEmpty()) { + lines.append(QStringLiteral("First heading: %1").arg(document.title)); + } + lines.append(QStringLiteral("Top-level blocks: %1").arg( + static_cast(document.document.children.size()))); + lines.append(QStringLiteral("Formulas: %1").arg( + static_cast(document.formulas.size()))); + lines.append(QStringLiteral("Warnings: %1").arg( + static_cast(document.warnings.size()))); + lines.append(QString()); + lines.append(QStringLiteral("Debug dump:")); + + const auto dumpLines = DumpForDebug(document).split('\n'); + const auto shown = (dumpLines.size() < qsizetype(kDiagnosticDumpLineCount)) + ? dumpLines.size() + : qsizetype(kDiagnosticDumpLineCount); + for (auto i = qsizetype(0); i != shown; ++i) { + lines.append(dumpLines[i]); + } + if (dumpLines.size() > shown) { + lines.append(QStringLiteral("...")); + } + return lines.join('\n'); +} + +} // namespace std::unique_ptr CreateMarkdownPreviewWidget( - const PreparedDocument &, - const OpenOptions &) { - return std::make_unique(); + const PreparedDocument &document, + const OpenOptions &options) { + auto root = std::make_unique(); + const auto scroll = Ui::CreateChild( + root.get(), + st::boxScroll); + auto label = object_ptr( + scroll, + BuildDiagnosticText(document, options), + st::aboutLabel); + const auto rawLabel = label.data(); + rawLabel->setSelectable(true); + rawLabel->setBreakEverywhere(true); + const auto wrap = scroll->setOwnedWidget( + object_ptr>( + scroll, + std::move(label), + st::boxPadding)); + + root->sizeValue() | rpl::on_next([=](QSize size) { + scroll->setGeometry(QRect(QPoint(), size)); + if (wrap) { + wrap->resizeToWidth(scroll->width()); + } + }, root->lifetime()); + + scroll->show(); + if (wrap) { + wrap->show(); + } + rawLabel->show(); + return root; } } // namespace Iv::Markdown diff --git a/Telegram/SourceFiles/tests/test_markdown_iv.cpp b/Telegram/SourceFiles/tests/test_markdown_iv.cpp new file mode 100644 index 0000000000..c8b33da13c --- /dev/null +++ b/Telegram/SourceFiles/tests/test_markdown_iv.cpp @@ -0,0 +1,502 @@ +#include "iv/markdown/iv_markdown_document.h" +#include "iv/markdown/iv_markdown_parse.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +using namespace Iv::Markdown; + +constexpr auto kValidationSourceLimit = 4 * 1024 * 1024; +constexpr auto kValidationFormulaLimit = 64 * 1024; + +struct Args { + QString markdownPath; + QString latexMarkdownPath; + bool dump = false; + bool ok = true; + QString error; +}; + +[[nodiscard]] QString FromLatin1(const char *value) { + return QString::fromLatin1(value); +} + +void PrintStreamLine(std::ostream &stream, const QString &line) { + const auto bytes = line.toUtf8(); + stream.write(bytes.constData(), static_cast(bytes.size())); + stream << '\n'; +} + +void PrintLine(const QString &line) { + PrintStreamLine(std::cout, line); +} + +void PrintError(const QString &line) { + PrintStreamLine(std::cerr, line); +} + +[[nodiscard]] Args ParseArgs(int argc, char **argv) { + auto result = Args(); + for (auto i = 1; i != argc; ++i) { + const auto argument = QString::fromLocal8Bit(argv[i]); + if (argument == FromLatin1("--dump")) { + result.dump = true; + } else if (argument == FromLatin1("--markdown") + || argument == FromLatin1("--latex-md")) { + if (i + 1 == argc) { + result.ok = false; + result.error = FromLatin1("missing value for ") + argument; + return result; + } + const auto path = QString::fromLocal8Bit(argv[++i]); + if (argument == FromLatin1("--markdown")) { + result.markdownPath = path; + } else { + result.latexMarkdownPath = path; + } + } else { + result.ok = false; + result.error = FromLatin1("unknown argument: ") + argument; + return result; + } + } + return result; +} + +[[nodiscard]] QString DefaultFixturePath(const QString &name) { + const auto applicationDir = QDir(QCoreApplication::applicationDirPath()); + const auto repoFixtureFromApplication = QDir::cleanPath( + applicationDir.filePath( + FromLatin1("../../Telegram/MarkdownMathProbes/fixtures/") + name)); + if (QFileInfo::exists(repoFixtureFromApplication)) { + return repoFixtureFromApplication; + } + const auto repoFixtureFromCurrent = QDir::current().filePath( + FromLatin1("Telegram/MarkdownMathProbes/fixtures/") + name); + if (QFileInfo::exists(repoFixtureFromCurrent)) { + return repoFixtureFromCurrent; + } + const auto applicationCandidate = applicationDir.filePath(name); + if (QFileInfo::exists(applicationCandidate)) { + return applicationCandidate; + } + const auto outDebug = QDir::current().filePath( + FromLatin1("out/Debug/") + name); + if (QFileInfo::exists(outDebug)) { + return outDebug; + } + return outDebug; +} + +[[nodiscard]] bool ReadFile(const QString &path, QByteArray *bytes) { + if (!bytes) { + return false; + } + auto file = QFile(path); + if (!file.open(QIODevice::ReadOnly)) { + return false; + } + *bytes = file.readAll(); + return true; +} + +[[nodiscard]] int CountNodes(const MarkdownNode &node) { + auto result = 1; + for (const auto &child : node.children) { + result += CountNodes(child); + } + return result; +} + +[[nodiscard]] bool HasKind(const MarkdownNode &node, NodeKind kind) { + if (node.kind == kind) { + return true; + } + for (const auto &child : node.children) { + if (HasKind(child, kind)) { + return true; + } + } + return false; +} + +[[nodiscard]] bool HasTaskState(const MarkdownNode &node, TaskState state) { + if (node.taskState == state) { + return true; + } + for (const auto &child : node.children) { + if (HasTaskState(child, state)) { + return true; + } + } + return false; +} + +[[nodiscard]] int CountDisplayMathNodes(const MarkdownNode &node) { + auto result = (node.kind == NodeKind::DisplayMath) ? 1 : 0; + for (const auto &child : node.children) { + result += CountDisplayMathNodes(child); + } + return result; +} + +[[nodiscard]] int CountFormulas( + const PreparedDocument &document, + MathKind kind) { + auto result = 0; + for (const auto &formula : document.formulas) { + if (formula.kind == kind) { + ++result; + } + } + return result; +} + +[[nodiscard]] bool HasFormulaOnLine( + const PreparedDocument &document, + int line, + const QString &tex) { + for (const auto &formula : document.formulas) { + if (formula.range.available + && formula.range.startLine == line + && formula.tex == tex) { + return true; + } + } + return false; +} + +[[nodiscard]] bool HasFormulaInLineRange( + const PreparedDocument &document, + int firstLine, + int lastLine) { + if (lastLine < firstLine) { + return false; + } + for (const auto &formula : document.formulas) { + if (!formula.range.available) { + continue; + } + if (formula.range.startLine <= lastLine + && formula.range.endLine >= firstLine) { + return true; + } + } + return false; +} + +[[nodiscard]] QString YesNo(bool value) { + return FromLatin1(value ? "yes" : "no"); +} + +[[nodiscard]] bool HasBothTaskStates(const PreparedDocument &document) { + return HasTaskState(document.document, TaskState::Checked) + && HasTaskState(document.document, TaskState::Unchecked); +} + +[[nodiscard]] bool ExclusionsPass(const PreparedDocument &document) { + return !HasFormulaInLineRange(document, 281, 281) + && HasFormulaOnLine(document, 285, FromLatin1("5x + 3")) + && !HasFormulaInLineRange(document, 332, 340); +} + +[[nodiscard]] bool HasFormula( + const PreparedDocument &document, + MathKind kind, + const QString &tex) { + for (const auto &formula : document.formulas) { + if (formula.kind == kind && formula.tex == tex) { + return true; + } + } + return false; +} + +void AppendSummaryCounts(QString *line, const PreparedDocument &document) { + line->append(FromLatin1(" nodes=")); + line->append(QString::number(document.stats.cmarkNodeCount)); + line->append(FromLatin1(" converted=")); + line->append(QString::number(CountNodes(document.document))); + line->append(FromLatin1(" formulas_inline=")); + line->append(QString::number(CountFormulas(document, MathKind::Inline))); + line->append(FromLatin1(" formulas_display=")); + line->append(QString::number(CountFormulas(document, MathKind::Display))); +} + +void PrintSummary(const PreparedDocument &document, const QString &label) { + auto line = label; + AppendSummaryCounts(&line, document); + line.append(FromLatin1(" tables=")); + line.append(YesNo(HasKind(document.document, NodeKind::Table))); + if (label == FromLatin1("markdown-example.md")) { + line.append(FromLatin1(" tasks=")); + line.append(YesNo(HasBothTaskStates(document))); + line.append(FromLatin1(" strike=")); + line.append(YesNo(HasKind(document.document, NodeKind::Strike))); + } else if (label == FromLatin1("latex-markdown-test.md")) { + line.append(FromLatin1(" exclusions=")); + line.append(YesNo(ExclusionsPass(document))); + } + PrintLine(line); +} + +[[nodiscard]] bool ParseFixture( + const QString &path, + const QString &label, + PreparedDocument *document) { + auto bytes = QByteArray(); + if (!ReadFile(path, &bytes)) { + PrintError(label + FromLatin1(" read-failed: ") + path); + return false; + } + auto parsed = ParseMarkdownForIv(bytes, ParseOptions{ label }); + if (!parsed.ok) { + PrintError(label + FromLatin1(" parse-failed: ") + parsed.error); + return false; + } + PrintSummary(parsed.document, label); + if (document) { + *document = std::move(parsed.document); + } + return true; +} + +void Check(bool condition, const QString &message, bool *ok) { + if (condition) { + return; + } + if (ok) { + *ok = false; + } + PrintError(FromLatin1("assertion failed: ") + message); +} + +void CheckParseSuccess( + const QByteArray &source, + const QString &label, + bool *ok) { + const auto parsed = ParseMarkdownForIv(source, ParseOptions{ label }); + Check( + parsed.ok, + label + FromLatin1(" parse failed: ") + parsed.error, + ok); +} + +void CheckParseFailure( + const QByteArray &source, + const QString &label, + const QString &expectedError, + bool *ok) { + const auto parsed = ParseMarkdownForIv(source, ParseOptions{ label }); + Check( + !parsed.ok, + label + FromLatin1(" should fail"), + ok); + if (!parsed.ok) { + Check( + parsed.error == expectedError, + label + FromLatin1(" error should be ") + + expectedError + + FromLatin1(", got ") + + parsed.error, + ok); + } +} + +void CheckValidationEdges(bool *ok) { + auto utf8BomSource = QByteArray::fromHex("EFBBBF"); + utf8BomSource.append("# Title\n"); + CheckParseSuccess( + utf8BomSource, + FromLatin1("utf8 bom"), + ok); + + CheckParseFailure( + QByteArray::fromHex("FFFE2300"), + FromLatin1("utf16 bom"), + FromLatin1("source-unsupported-bom"), + ok); + CheckParseFailure( + QByteArray("a\0b", 3), + FromLatin1("nul byte"), + FromLatin1("source-binary"), + ok); + CheckParseFailure( + QByteArray::fromHex("C328"), + FromLatin1("invalid utf8"), + FromLatin1("source-invalid-utf8"), + ok); + + const auto oversizedSource = QByteArray(kValidationSourceLimit + 1, 'a'); + CheckParseFailure( + oversizedSource, + FromLatin1("source size"), + FromLatin1("source-too-large"), + ok); + + auto oversizedFormula = QByteArray(); + oversizedFormula.reserve(kValidationFormulaLimit + 2); + oversizedFormula.append('$'); + oversizedFormula.append(QByteArray(kValidationFormulaLimit + 1, '+')); + oversizedFormula.append('$'); + CheckParseFailure( + oversizedFormula, + FromLatin1("formula size"), + FromLatin1("formula-too-large"), + ok); + + const auto generated = QByteArray( + "Inline code `$code$`.\n" + "```\n" + "$block$\n" + "```\n" + "Escaped \\$ and price $5.99$.\n" + "Real $x + y$ done.\n"); + const auto parsed = ParseMarkdownForIv( + generated, + ParseOptions{ FromLatin1("generated-edge-checks.md") }); + Check( + parsed.ok, + FromLatin1("generated exclusions parse failed: ") + parsed.error, + ok); + if (parsed.ok) { + Check( + static_cast(parsed.document.formulas.size()) == 1, + FromLatin1("generated exclusions formula count"), + ok); + Check( + CountFormulas(parsed.document, MathKind::Inline) == 1, + FromLatin1("generated exclusions inline formula count"), + ok); + Check( + CountFormulas(parsed.document, MathKind::Display) == 0, + FromLatin1("generated exclusions display formula count"), + ok); + Check( + HasFormula(parsed.document, MathKind::Inline, FromLatin1("x + y")), + FromLatin1("generated exclusions real formula"), + ok); + } +} + +} // namespace + +int main(int argc, char **argv) { + auto application = QCoreApplication(argc, argv); + (void)application; + + auto args = ParseArgs(argc, argv); + if (!args.ok) { + PrintError(args.error); + return 1; + } + if (args.markdownPath.isEmpty()) { + args.markdownPath = DefaultFixturePath(FromLatin1("markdown-example.md")); + } + if (args.latexMarkdownPath.isEmpty()) { + args.latexMarkdownPath = DefaultFixturePath( + FromLatin1("latex-markdown-test.md")); + } + + auto markdown = PreparedDocument(); + if (!ParseFixture( + args.markdownPath, + FromLatin1("markdown-example.md"), + &markdown)) { + return 1; + } + if (args.dump) { + PrintLine(DumpForDebug(markdown)); + } + + auto latex = PreparedDocument(); + if (!ParseFixture( + args.latexMarkdownPath, + FromLatin1("latex-markdown-test.md"), + &latex)) { + return 1; + } + if (args.dump) { + PrintLine(DumpForDebug(latex)); + } + + auto ok = true; + Check( + markdown.stats.cmarkNodeCount == 562, + FromLatin1("markdown-example.md cmark node count"), + &ok); + Check( + CountFormulas(markdown, MathKind::Inline) == 1, + FromLatin1("markdown-example.md inline formula count"), + &ok); + Check( + CountFormulas(markdown, MathKind::Display) == 1, + FromLatin1("markdown-example.md display formula count"), + &ok); + Check( + CountDisplayMathNodes(markdown.document) == 1, + FromLatin1("markdown-example.md display math node count"), + &ok); + Check( + HasKind(markdown.document, NodeKind::Table), + FromLatin1("markdown-example.md table coverage"), + &ok); + Check( + HasKind(markdown.document, NodeKind::Strike), + FromLatin1("markdown-example.md strikethrough coverage"), + &ok); + Check( + HasTaskState(markdown.document, TaskState::Checked), + FromLatin1("markdown-example.md checked task"), + &ok); + Check( + HasTaskState(markdown.document, TaskState::Unchecked), + FromLatin1("markdown-example.md unchecked task"), + &ok); + Check( + latex.stats.cmarkNodeCount == 532, + FromLatin1("latex-markdown-test.md cmark node count"), + &ok); + Check( + CountFormulas(latex, MathKind::Inline) == 99, + FromLatin1("latex-markdown-test.md inline formula count"), + &ok); + Check( + CountFormulas(latex, MathKind::Display) == 31, + FromLatin1("latex-markdown-test.md display formula count"), + &ok); + Check( + CountDisplayMathNodes(latex.document) == 31, + FromLatin1("latex-markdown-test.md display math node count"), + &ok); + Check( + HasKind(latex.document, NodeKind::Table), + FromLatin1("latex-markdown-test.md table coverage"), + &ok); + Check( + !HasFormulaInLineRange(latex, 281, 281), + FromLatin1("latex-markdown-test.md line 281 exclusion"), + &ok); + Check( + HasFormulaOnLine(latex, 285, FromLatin1("5x + 3")), + FromLatin1("latex-markdown-test.md line 285 formula"), + &ok); + Check( + !HasFormulaInLineRange(latex, 332, 340), + FromLatin1("latex-markdown-test.md lines 332-340 exclusions"), + &ok); + + CheckValidationEdges(&ok); + + return ok ? 0 : 1; +} diff --git a/Telegram/cmake/td_iv.cmake b/Telegram/cmake/td_iv.cmake index b1e0a04095..913b51af10 100644 --- a/Telegram/cmake/td_iv.cmake +++ b/Telegram/cmake/td_iv.cmake @@ -31,6 +31,8 @@ if (TDESKTOP_NATIVE_MARKDOWN_IV) iv/markdown/iv_markdown_controller.h iv/markdown/iv_markdown_document.cpp iv/markdown/iv_markdown_document.h + iv/markdown/iv_markdown_math.cpp + iv/markdown/iv_markdown_math.h iv/markdown/iv_markdown_microtex.cpp iv/markdown/iv_markdown_microtex.h iv/markdown/iv_markdown_parse.cpp diff --git a/Telegram/cmake/telegram_options.cmake b/Telegram/cmake/telegram_options.cmake index 14d8a040e4..1e7c5afe17 100644 --- a/Telegram/cmake/telegram_options.cmake +++ b/Telegram/cmake/telegram_options.cmake @@ -53,6 +53,10 @@ if (DESKTOP_APP_SPECIAL_TARGET) target_compile_definitions(Telegram PRIVATE TDESKTOP_ALLOW_CLOSED_ALPHA) endif() +if (TDESKTOP_NATIVE_MARKDOWN_IV) + target_compile_definitions(Telegram PRIVATE TDESKTOP_NATIVE_MARKDOWN_IV) +endif() + option(DESKTOP_APP_DISABLE_SWIFT6 "Disable local on-device translation (build without Swift 6 on macOS)." OFF) if (DESKTOP_APP_DISABLE_SWIFT6) target_compile_definitions(Telegram PRIVATE TDESKTOP_DISABLE_SWIFT6) diff --git a/Telegram/cmake/tests.cmake b/Telegram/cmake/tests.cmake index 4f2e5651b6..13e9bb9ebf 100644 --- a/Telegram/cmake/tests.cmake +++ b/Telegram/cmake/tests.cmake @@ -42,3 +42,39 @@ set_target_properties(test_text PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINA add_dependencies(Telegram test_text) target_prepare_qrc(test_text) + +if (TDESKTOP_NATIVE_MARKDOWN_IV) + add_executable(test_markdown_iv) + init_target(test_markdown_iv "(tests)") + + target_include_directories(test_markdown_iv PRIVATE ${src_loc}) + + nice_target_sources(test_markdown_iv ${src_loc} + PRIVATE + tests/test_markdown_iv.cpp + iv/markdown/iv_markdown_common.cpp + iv/markdown/iv_markdown_common.h + iv/markdown/iv_markdown_document.cpp + iv/markdown/iv_markdown_document.h + iv/markdown/iv_markdown_math.cpp + iv/markdown/iv_markdown_math.h + iv/markdown/iv_markdown_parse.cpp + iv/markdown/iv_markdown_parse.h + ) + + target_compile_definitions(test_markdown_iv + PRIVATE + TDESKTOP_NATIVE_MARKDOWN_IV + ) + + target_link_libraries(test_markdown_iv + PRIVATE + desktop-app::external_cmark_gfm + desktop-app::external_qt + desktop-app::lib_base + ) + + set_target_properties(test_markdown_iv PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) + + add_dependencies(Telegram test_markdown_iv) +endif()