Удаление невидимых символов из w:t: FOP больше не рисует пустые строки и боксы
PR checks / web (pull_request) Successful in 39s
PR checks / java (pull_request) Successful in 58s
Main: tests, deploy, autotests / web (push) Successful in 41s
Main: tests, deploy, autotests / java (push) Successful in 53s
Main: tests, deploy, autotests / images-web (push) Successful in 32s
Main: tests, deploy, autotests / images-app (push) Successful in 3m50s
Main: tests, deploy, autotests / deploy (push) Successful in 1m31s
Main: tests, deploy, autotests / autotests (push) Successful in 49s
Main: tests, deploy, autotests / rollback (push) Has been skipped
PR checks / web (pull_request) Successful in 39s
PR checks / java (pull_request) Successful in 58s
Main: tests, deploy, autotests / web (push) Successful in 41s
Main: tests, deploy, autotests / java (push) Successful in 53s
Main: tests, deploy, autotests / images-web (push) Successful in 32s
Main: tests, deploy, autotests / images-app (push) Successful in 3m50s
Main: tests, deploy, autotests / deploy (push) Successful in 1m31s
Main: tests, deploy, autotests / autotests (push) Successful in 49s
Main: tests, deploy, autotests / rollback (push) Has been skipped
This commit was merged in pull request #38.
This commit is contained in:
@@ -575,6 +575,125 @@ class WordPdfConverterTest {
|
||||
assertTrue(text.contains("таблица"), "текст PDF: " + text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Абзац только из невидимых символов (U+0085/U+2028/U+2029/U+2060/U+FEFF)
|
||||
* не раздувает PDF: FOP превращает первые три в принудительные переносы
|
||||
* (CharUtilities.isExplicitBreak) — до InvisibleCharNormalizer между
|
||||
* маркерами вставало ≥2 пустых строк. Регрессия — InvisibleCharNormalizer.
|
||||
*/
|
||||
@Test
|
||||
void invisibleChars_noEmptyLines() throws Exception {
|
||||
String inv = invisible(0x0085, 0x2028, 0x2029, 0x2060, 0xFEFF);
|
||||
byte[] docx = minimalDocx(
|
||||
"<w:p><w:r><w:t>AAA1</w:t></w:r></w:p>"
|
||||
+ "<w:p><w:r><w:t>CCC1</w:t></w:r></w:p>"
|
||||
+ "<w:p><w:r><w:t>" + inv + "</w:t></w:r></w:p>"
|
||||
+ "<w:p><w:r><w:t>BBB1</w:t></w:r></w:p>");
|
||||
|
||||
Path pdf = convertPdf(docx, "invisible");
|
||||
String text = pdfText(pdf);
|
||||
String norm = norm(text);
|
||||
assertTrue(norm.contains("AAA1"), "текст PDF: " + norm);
|
||||
assertTrue(norm.contains("CCC1"), "текст PDF: " + norm);
|
||||
assertTrue(norm.contains("BBB1"), "текст PDF: " + norm);
|
||||
for (int cp : new int[] {0x0085, 0x2028, 0x2029, 0x2060, 0xFEFF}) {
|
||||
assertTrue(text.indexOf(cp) < 0,
|
||||
"в PDF остался код U+" + Integer.toHexString(cp));
|
||||
}
|
||||
// самокалибровка: шаг AAA1→CCC1 = одна строка; между CCC1 и BBB1 —
|
||||
// абзац из невидимых, который после вырезания это честная пустая
|
||||
// строка (итог 2 шага). С багом FOP добавлял переносы — ≥3 шагов
|
||||
// (маркеры — первые символы: PDFTextStripper режет ран на части)
|
||||
double line = yOfMarker(pdf, 'C') - yOfMarker(pdf, 'A');
|
||||
double gap = yOfMarker(pdf, 'B') - yOfMarker(pdf, 'C');
|
||||
assertTrue(gap <= 2.5 * line,
|
||||
"пустые строки: gap=" + gap + " pt при строке " + line + " pt");
|
||||
}
|
||||
|
||||
/**
|
||||
* Невидимые знаки в ранах Arial (Arimo рисует для U+2028/U+FEFF видимые
|
||||
* боксы, cmap → ненулевой GID) вырезаются: в PDF склеенный текст без
|
||||
* боксов. Регрессия — InvisibleCharNormalizer.
|
||||
*/
|
||||
@Test
|
||||
void invisibleChars_arimoBoxesNotDrawn() throws Exception {
|
||||
String arial = "<w:rPr><w:rFonts w:ascii=\"Arial\" w:hAnsi=\"Arial\"/></w:rPr>";
|
||||
byte[] docx = minimalDocx(
|
||||
"<w:p><w:r>" + arial + "<w:t>X" + invisible(0x2028)
|
||||
+ "Y</w:t></w:r></w:p>"
|
||||
+ "<w:p><w:r>" + arial + "<w:t>Z" + invisible(0xFEFF)
|
||||
+ "W</w:t></w:r></w:p>");
|
||||
|
||||
Path pdf = convertPdf(docx, "arimo-boxes");
|
||||
String text = pdfText(pdf);
|
||||
String norm = norm(text);
|
||||
assertTrue(norm.contains("XY"), "текст PDF: " + norm);
|
||||
assertTrue(norm.contains("ZW"), "текст PDF: " + norm);
|
||||
for (int cp : new int[] {0x2028, 0xFEFF}) {
|
||||
assertTrue(text.indexOf(cp) < 0,
|
||||
"в PDF остался код U+" + Integer.toHexString(cp));
|
||||
}
|
||||
}
|
||||
|
||||
/** Строка из кодовых точек (невидимые литералы в исходнике нечитаемы). */
|
||||
private static String invisible(int... cps) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int cp : cps) {
|
||||
sb.appendCodePoint(cp);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** Y базовой линии первого TextPosition, содержащего символ {@code ch}. */
|
||||
private static double yOfMarker(Path pdf, char ch) throws IOException {
|
||||
try (PDDocument doc = Loader.loadPDF(pdf.toFile())) {
|
||||
double[] found = {Double.NaN};
|
||||
PDFTextStripper stripper = new PDFTextStripper() {
|
||||
@Override
|
||||
protected void processTextPosition(
|
||||
org.apache.pdfbox.text.TextPosition position) {
|
||||
if (Double.isNaN(found[0])
|
||||
&& position.getUnicode().indexOf(ch) >= 0) {
|
||||
found[0] = position.getYDirAdj();
|
||||
}
|
||||
super.processTextPosition(position);
|
||||
}
|
||||
};
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(doc.getNumberOfPages());
|
||||
stripper.getText(doc);
|
||||
assertFalse(Double.isNaN(found[0]), "в PDF нет символа «" + ch + "»");
|
||||
return found[0];
|
||||
}
|
||||
}
|
||||
|
||||
/** Минимальный transitional-docx с указанным телом w:body. */
|
||||
private static byte[] minimalDocx(String bodyXml) throws IOException {
|
||||
String w = "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zos = new ZipOutputStream(out)) {
|
||||
putZip(zos, "[Content_Types].xml",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"
|
||||
+ "<Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\">"
|
||||
+ "<Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\"/>"
|
||||
+ "<Default Extension=\"xml\" ContentType=\"application/xml\"/>"
|
||||
+ "<Override PartName=\"/word/document.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml\"/>"
|
||||
+ "</Types>");
|
||||
putZip(zos, "_rels/.rels",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"
|
||||
+ "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">"
|
||||
+ "<Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument\" Target=\"word/document.xml\"/>"
|
||||
+ "</Relationships>");
|
||||
putZip(zos, "word/document.xml",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"
|
||||
+ "<w:document xmlns:w=\"" + w + "\"><w:body>"
|
||||
+ bodyXml
|
||||
+ "<w:sectPr><w:pgSz w:w=\"11906\" w:h=\"16838\"/></w:sectPr>"
|
||||
+ "</w:body></w:document>");
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private static void putZip(ZipOutputStream zos, String name, String xml) throws IOException {
|
||||
zos.putNextEntry(new ZipEntry(name));
|
||||
zos.write(xml.getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
package ru.pjandro.wordpdf.internal;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
|
||||
/**
|
||||
* Вырезание невидимых символов из {@code w:t} (см. InvisibleCharNormalizer):
|
||||
* FOP превращает U+0085/U+2028/U+2029 в принудительные переносы, а
|
||||
* Arimo/Tinos рисуют боксы для U+2060/U+FEFF/bidi-меток.
|
||||
*
|
||||
* Невидимые коды собираются через {@link #s(int...)}: литералы с ними
|
||||
* нечитаемы, а Java-препроцессор ломает {@code \u0085}/{@code \u2028}/
|
||||
* {@code \u2029} в исходнике (это line terminator'ы языка).
|
||||
*/
|
||||
class InvisibleCharNormalizerTest {
|
||||
|
||||
private static final int NEL = 0x0085;
|
||||
private static final int LS = 0x2028;
|
||||
private static final int PS = 0x2029;
|
||||
private static final int LRM = 0x200E;
|
||||
private static final int RLM = 0x200F;
|
||||
private static final int LRE = 0x202A;
|
||||
private static final int RLE = 0x202B;
|
||||
private static final int PDF = 0x202C;
|
||||
private static final int LRO = 0x202D;
|
||||
private static final int RLO = 0x202E;
|
||||
private static final int WJ = 0x2060;
|
||||
private static final int ZWNBSP = 0xFEFF;
|
||||
private static final int NBSP = 0x00A0;
|
||||
private static final int SHY = 0x00AD;
|
||||
private static final int ZWSP = 0x200B;
|
||||
private static final int ZWNJ = 0x200C;
|
||||
private static final int ZWJ = 0x200D;
|
||||
|
||||
/** Строка из кодовых точек. */
|
||||
private static String s(int... cps) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int cp : cps) {
|
||||
sb.appendCodePoint(cp);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static byte[] makeDocx(String... namesAndXml) throws IOException {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zos = new ZipOutputStream(out)) {
|
||||
for (int i = 0; i < namesAndXml.length; i += 2) {
|
||||
zos.putNextEntry(new ZipEntry(namesAndXml[i]));
|
||||
zos.write(namesAndXml[i + 1].getBytes(StandardCharsets.UTF_8));
|
||||
zos.closeEntry();
|
||||
}
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private static String readEntry(byte[] zip, String name) throws IOException {
|
||||
try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(zip))) {
|
||||
ZipEntry e;
|
||||
while ((e = zis.getNextEntry()) != null) {
|
||||
if (e.getName().equals(name)) {
|
||||
return new String(zis.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new AssertionError("нет записи " + name);
|
||||
}
|
||||
|
||||
@Test
|
||||
void stripsSeparatorsAndBidiMarks() {
|
||||
String r = InvisibleCharNormalizer.rewrite("<w:r><w:t>А"
|
||||
+ s(NEL) + "Б" + s(LS) + "В" + s(PS) + "Г" + s(WJ) + "Д"
|
||||
+ s(ZWNBSP) + "Е" + s(LRM) + "Ж" + s(RLM) + "З" + s(LRE)
|
||||
+ "И" + s(RLE) + "К" + s(PDF) + "Л" + s(LRO) + "М" + s(RLO)
|
||||
+ "N</w:t></w:r>");
|
||||
assertEquals("<w:r><w:t>АБВГДЕЖЗИКЛМN</w:t></w:r>", r);
|
||||
}
|
||||
|
||||
@Test
|
||||
void numericEntitiesRemoved() {
|
||||
String r = InvisibleCharNormalizer.rewrite(
|
||||
"<w:t>а
б
в…гд⁠е⁠ё"
|
||||
+ "‎ж‏з‪и‫к‬л‭м</w:t>");
|
||||
assertEquals("<w:t>абвгдеёжзиклм</w:t>", r);
|
||||
}
|
||||
|
||||
@Test
|
||||
void zeroWidthAndSoftHyphenKept() {
|
||||
// U+00A0, U+00AD, U+200B, U+200C, U+200D — невидимы во всех шрифтах
|
||||
// бандла и значимы (перенос/shaping): не трогаем
|
||||
assertNull(InvisibleCharNormalizer.rewrite(
|
||||
"<w:t>а" + s(NBSP) + "б" + s(SHY) + "в" + s(ZWSP)
|
||||
+ "г" + s(ZWNJ) + "д" + s(ZWJ) + "е</w:t>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rewriteWithoutMatchesIsNull() {
|
||||
assertNull(InvisibleCharNormalizer.rewrite(
|
||||
"<w:r><w:t>обычный текст — с тире</w:t></w:r>"));
|
||||
// коды набора вне тела w:t (чужие теги) не трогаем
|
||||
assertNull(InvisibleCharNormalizer.rewrite(
|
||||
"<w:instrText>PAGE" + s(LS) + "</w:instrText>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nothingToChangeReturnsOriginalArray() throws Exception {
|
||||
byte[] docx = makeDocx(
|
||||
"[Content_Types].xml", "<Types/>",
|
||||
"word/document.xml",
|
||||
"<w:document><w:p><w:r><w:t>чисто</w:t></w:r></w:p></w:document>");
|
||||
|
||||
assertSame(docx, InvisibleCharNormalizer.normalize(docx));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rewritesWordPartsInZip() throws Exception {
|
||||
byte[] docx = makeDocx(
|
||||
"[Content_Types].xml", "<Types/>",
|
||||
"word/document.xml",
|
||||
"<w:document><w:p><w:r><w:t>А" + s(LS) + "Б" + s(NEL)
|
||||
+ "В" + s(ZWNBSP) + "Г</w:t></w:r></w:p></w:document>");
|
||||
|
||||
String document = readEntry(InvisibleCharNormalizer.normalize(docx),
|
||||
"word/document.xml");
|
||||
assertEquals("<w:document><w:p><w:r><w:t>АБВГ</w:t></w:r></w:p></w:document>",
|
||||
document);
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonWordPartsNotTouched() throws Exception {
|
||||
byte[] docx = makeDocx(
|
||||
"[Content_Types].xml", "<Types>" + s(LS) + "</Types>",
|
||||
"word/document.xml", "<w:document/>");
|
||||
|
||||
byte[] result = InvisibleCharNormalizer.normalize(docx);
|
||||
assertEquals("<Types>" + s(LS) + "</Types>",
|
||||
readEntry(result, "[Content_Types].xml"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user