Some compile output still leaks through with --verbosity=1

  • Open
  • quality assurance status badge
Details
3 participants
  • Sarah Morgensen
  • Ludovic Courtès
  • Mark H Weaver
Owner
unassigned
Submitted by
Mark H Weaver
Severity
important
M
M
Mark H Weaver wrote on 20 Apr 2019 16:53
(address . bug-guix@gnu.org)
87mukkfd2j.fsf@netris.org
Sometimes when compiling a package with --verbosity=1, some parts of the
compile output leak through. For example, see the transcript below.

Mark


Toggle snippet (278 lines)
mhw@jojen ~$ guix system build /etc/config.scm --verbosity=1 --keep-failed --keep-going; guix package -m mhw-manifest.scm --keep-failed --keep-going
building /gnu/store/3c9f2rv92fb5id7xy0h62jl4vvglvpvh-linux-libre-5.0.8-gnu.tar.xz.drv...
downloading from https://linux-libre.fsfla.org/pub/linux-libre/releases/5.0.8-gnu/linux-libre-5.0.8-gnu.tar.xz...
building /gnu/store/cxlivh8bq6adnhdzc6i75z5i2qplq8zr-linux-libre-5.0.8-gnu.tar.xz.drv...
building /gnu/store/9xlx7zj4w9cl69xi3d21gl40l3gx4jri-linux-libre-5.0.8.drv...
building /gnu/store/la179xxpm8m4jmajjr3xfrchzfwpc06y-console-setup-1.191-checkout.drv...
building /gnu/store/v7a1y4v0nwrq64ddr7kdm4855gzjrzrc-console-setup-1.191.drv...
building /gnu/store/04wdi408fbl9bmnzdy4dwxd135cy9jss-gpgme-1.13.0.tar.bz2.drv...
downloading from https://gnupg.org/ftp/gcrypt/gpgme/gpgme-1.13.0.tar.bz2...
building /gnu/store/yaavzrys958mhmki2axpdnbrlyzd911f-gpgme-1.13.0.drv...
- 'check' phaselt.
building /gnu/store/0rdfdqsahs0axzhpn6k3g4zmskmgkvcg-samba-4.10.2.drv...
building /gnu/store/75z0i5hb2cwn0y76gnaqlj18z1715yy1-volume-key-0.3.12.drv...
building /gnu/store/0fjcqx6s7bm7wsa4iwf2knpki9r8g71h-libblockdev-2.21.drv...
building /gnu/store/dk1867ygsaimm5yzxp6rrx3379fsx878-udisks-2.7.7.drv...
building /gnu/store/a2cwdflfbklmj4cflrzkd4n4m3q3ain5-gvfs-1.32.1.drv...
building /gnu/store/j8a8k76qaslj3bvbbbzpp6wwh80cv27b-libmbim-1.18.2.tar.xz.drv...
downloading from https://www.freedesktop.org/software/libmbim/libmbim-1.18.2.tar.xz...
building /gnu/store/070338avjl034y1vqd40hb62aw3i7yg0-libqmi-1.22.4.tar.xz.drv...
downloading from https://www.freedesktop.org/software/libqmi/libqmi-1.22.4.tar.xz...
building /gnu/store/vgli6q8lyfp92mxr9v5783kd2cgyb3mr-libmbim-1.18.2.drv...
building /gnu/store/7jq010r8fvdgdr9sb6g2znrkqkjgpbjk-libqmi-1.22.4.drv...
building /gnu/store/rn5mbq2h2fmxpd95q6cfx31bgbw5q89a-modem-manager-1.4.14.drv...
\ 'build' phase�@ build-log 30033 4096
� is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_error (ctx->result,
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:122:21: note: declared here
void g_simple_async_result_set_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6136:5: warning: ‘g_simple_async_result_set_op_res_gboolean’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_op_res_gboolean (ctx->result, TRUE);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:88:21: note: declared here
void g_simple_async_result_set_op_res_gboolean (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c: In function ‘sms_pdu_part_list_ready’:
mm-broadband-modem.c:6172:9: warning: ‘g_simple_async_result_take_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_take_error (ctx->result, error);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:116:21: note: declared here
void g_simple_async_result_take_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6179:9: warning: ‘g_simple_async_result_take_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_take_error (ctx->result, error);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:116:21: note: declared here
void g_simple_async_result_take_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6205:5: warning: ‘g_simple_async_result_set_op_res_gboolean’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_op_res_gboolean (ctx->result, TRUE);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:88:21: note: declared here
void g_simple_async_result_set_op_res_gboolean (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c: In function ‘list_parts_lock_storages_ready’:
mm-broadband-modem.c:6219:9: warning: ‘g_simple_async_result_take_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_take_error (ctx->result, error);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
@ build-log 30033 4096
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:116:21: note: declared here
void g_simple_async_result_take_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c: In function ‘modem_messaging_load_initial_sms_parts’:
mm-broadband-modem.c:6249:15: warning: assignment from incompatible pointer type [-Wincompatible-pointer-types]
ctx->self = g_object_ref (self);
^
mm-broadband-modem.c:6250:5: warning: ‘g_simple_async_result_new’ is deprecated: Use 'g_task_new' instead [-Wdeprecated-declarations]
ctx->result = g_simple_async_result_new (G_OBJECT (self),
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:51:21: note: declared here
GSimpleAsyncResult *g_simple_async_result_new (GObject *source_object,
^
mm-broadband-modem.c: In function ‘hdr_state_context_complete_and_free’:
mm-broadband-modem.c:6365:5: warning: ‘g_simple_async_result_complete’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_complete (ctx->result);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:104:21: note: declared here
void g_simple_async_result_complete (GSimpleAsyncResult *simple);
^
mm-broadband-modem.c: In function ‘modem_cdma_get_hdr_state_finish’:
mm-broadband-modem.c:6382:5: warning: ‘g_simple_async_result_propagate_error’ is deprecated [-Wdeprecated-declarations]
if (g_simple_async_result_propagate_error (G_SIMPLE_ASYNC_RESULT (res), error))
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:119:21: note: declared here
gboolean g_simple_async_result_propagate_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6385:5: warning: ‘g_simple_async_result_get_op_res_gpointer’ is deprecated [-Wdeprecated-declarations]
results = g_simple_async_result_get_op_res_gpointer (G_SIMPLE_ASYNC_RESULT (res));
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:79:21: note: declared here
gpointer g_simple_async_result_get_op_res_gpointer (GSimpleAsyncResult *simple);
^
mm-broadband-modem.c: In function ‘hdr_subsys_state_info_ready’:
mm-broadband-modem.c:6405:9: warning: ‘g_simple_async_result_set_from_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_from_error (ctx->result, error);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-g@ build-log 30033 4096
lib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:113:21: note: declared here
void g_simple_async_result_set_from_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6415:9: warning: ‘g_simple_async_result_set_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_error (ctx->result,
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:122:21: note: declared here
void g_simple_async_result_set_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6433:5: warning: ‘g_simple_async_result_set_op_res_gpointer’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_op_res_gpointer (ctx->result, results, (GDestroyNotify)g_free);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:75:21: note: declared here
void g_simple_async_result_set_op_res_gpointer (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c: In function ‘modem_cdma_get_hdr_state’:
mm-broadband-modem.c:6448:9: warning: ‘g_simple_async_report_error_in_idle’ is deprecated: Use 'g_task_report_error' instead [-Wdeprecated-declarations]
g_simple_async_report_error_in_idle (G_OBJECT (self),
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:140:6: note: declared here
void g_simple_async_report_error_in_idle (GObject *object,
^
mm-broadband-modem.c:6459:15: warning: assignment from incompatible pointer type [-Wincompatible-pointer-types]
ctx->self = g_object_ref (self);
^
mm-broadband-modem.c:6460:5: warning: ‘g_simple_async_result_new’ is deprecated: Use 'g_task_new' instead [-Wdeprecated-declarations]
ctx->result = g_simple_async_result_new (G_OBJECT (self),
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:51:21: note: declared here
GSimpleAsyncResult *g_simple_async_result_new (GObject *source_object,
^
mm-broadband-modem.c: In function ‘call_manager_state_context_complete_and_free’:
mm-broadband-modem.c:6497:5: warning: ‘g_simple_async_result_complete’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_complete (ctx->result);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncre@ build-log 30033 4096
sult.h:104:21: note: declared here
void g_simple_async_result_complete (GSimpleAsyncResult *simple);
^
mm-broadband-modem.c: In function ‘modem_cdma_get_call_manager_state_finish’:
mm-broadband-modem.c:6513:5: warning: ‘g_simple_async_result_propagate_error’ is deprecated [-Wdeprecated-declarations]
if (g_simple_async_result_propagate_error (G_SIMPLE_ASYNC_RESULT (res), error))
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:119:21: note: declared here
gboolean g_simple_async_result_propagate_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6516:5: warning: ‘g_simple_async_result_get_op_res_gpointer’ is deprecated [-Wdeprecated-declarations]
results = g_simple_async_result_get_op_res_gpointer (G_SIMPLE_ASYNC_RESULT (res));
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:79:21: note: declared here
gpointer g_simple_async_result_get_op_res_gpointer (GSimpleAsyncResult *simple);
^
mm-broadband-modem.c: In function ‘cm_subsys_state_info_ready’:
mm-broadband-modem.c:6535:9: warning: ‘g_simple_async_result_take_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_take_error (ctx->result, error);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:116:21: note: declared here
void g_simple_async_result_take_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6546:9: warning: ‘g_simple_async_result_set_error’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_error (ctx->result,
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:122:21: note: declared here
void g_simple_async_result_set_error (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c:6561:5: warning: ‘g_simple_async_result_set_op_res_gpointer’ is deprecated [-Wdeprecated-declarations]
g_simple_async_result_set_op_res_gpointer (ctx->result, results, (GDestroyNotify)g_free);
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:75:21: note: declared here
void g_simple_async_result_set_op_res_gpointer (GSimpleAsyncResult *simple,
^
mm-broadband-modem.c: In function ‘modem_cdma_get_call_manager_state’:
mm-broadband-modem.c:6576:9: warning: ‘g_sim@ build-log 30033 4096
ple_async_report_error_in_idle’ is deprecated: Use 'g_task_report_error' instead [-Wdeprecated-declarations]
g_simple_async_report_error_in_idle (G_OBJECT (self),
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:140:6: note: declared here
void g_simple_async_report_error_in_idle (GObject *object,
^
mm-broadband-modem.c:6587:15: warning: assignment from incompatible pointer type [-Wincompatible-pointer-types]
ctx->self = g_object_ref (self);
^
mm-broadband-modem.c:6588:5: warning: ‘g_simple_async_result_new’ is deprecated: Use 'g_task_new' instead [-Wdeprecated-declarations]
ctx->result = g_simple_async_result_new (G_OBJECT (self),
^
In file included from /gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gio.h:117:0,
from ../libmm-glib/generated/mm-gdbus-manager.h:11,
from ../libmm-glib/libmm-glib.h:77,
from mm-broadband-modem.c:27:
/gnu/store/cgy82g6yv8l1chawgch47zh23b0jll3l-glib-2.56.3/include/glib-2.0/gio/gsimpleasyncresult.h:51:21: note: declared here
GSimpleAsyncResult *g_simple_async_result_new (GObjec
This message was truncated. Download the full message here.
L
L
Ludovic Courtès wrote on 21 Apr 2019 13:15
(name . Mark H Weaver)(address . mhw@netris.org)(address . 35350@debbugs.gnu.org)
87r29v2jz2.fsf@gnu.org
Hi Mark,

Mark H Weaver <mhw@netris.org> skribis:

Toggle quote (3 lines)
> Sometimes when compiling a package with --verbosity=1, some parts of the
> compile output leak through. For example, see the transcript below.

Weird. What’s the value of --max-jobs? Is offloading enabled?

Presumably both the client and daemon are recent, right?

Thanks,
Ludo’.
M
M
Mark H Weaver wrote on 22 Apr 2019 16:52
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
87ftq9silk.fsf@netris.org
Hi Ludovic,

Ludovic Courtès <ludo@gnu.org> writes:

Toggle quote (7 lines)
> Mark H Weaver <mhw@netris.org> skribis:
>
>> Sometimes when compiling a package with --verbosity=1, some parts of the
>> compile output leak through. For example, see the transcript below.
>
> Weird.

FWIW, a few observations, possibly relevant:

(1) Each chunk of leaked output begins with 1 or 2 Unicode Replacement
characters (U+FFFD). In the transcript I provided, the first leak
began with 1 replacement char, and all later leaks began with 2.

(2) The replacement characters are immediately followed by
"@ build-log 30033 4096\n", and that string is also sprinkled
throughout the leaked output, with approximately ~4060-4070
characters of leaked output between each occurrence of
"@ build-log 30033 4096\n".

Toggle quote (2 lines)
> What’s the value of --max-jobs?

I'm not sure, how do I find out? I don't specify it in my OS
configuration, and it's not on the command line of 'guix-daemon'.
Here's the relevant portion of my OS config:

(modify-services %base-services
(guix-service-type config =>
(guix-configuration
(inherit config)
(use-substitutes? #f)
(authorized-keys '())
(substitute-urls '())
(extra-options '("--gc-keep-derivations=yes"
"--gc-keep-outputs=yes")))))

I guess it's probably 2, because that's how many cores are in this
machine (Thinkpad X200), and 2 is also the number of GCC processes that
I typically see when building packages on this machine.

Toggle quote (2 lines)
> Is offloading enabled?

No. In addition to the above configuration, my /etc/guix/acl also
contains simply "(acl)".

Toggle quote (2 lines)
> Presumably both the client and daemon are recent, right?

Yes, this machine is rarely more than a week out-of-date w.r.t. our
'master' branch.

Thanks,
Mark
M
M
Mark H Weaver wrote on 23 Apr 2019 01:45
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
877eblw1mz.fsf@netris.org
Toggle quote (2 lines)
> What’s the value of --max-jobs?

max-jobs is 1. Please disregard my previous response.

Thanks,
Mark
L
L
Ludovic Courtès wrote on 23 Apr 2019 03:12
(name . Mark H Weaver)(address . mhw@netris.org)(address . 35350@debbugs.gnu.org)
87imv5jai5.fsf@gnu.org
Hi Mark,

Mark H Weaver <mhw@netris.org> skribis:

Toggle quote (21 lines)
> Ludovic Courtès <ludo@gnu.org> writes:
>
>> Mark H Weaver <mhw@netris.org> skribis:
>>
>>> Sometimes when compiling a package with --verbosity=1, some parts of the
>>> compile output leak through. For example, see the transcript below.
>>
>> Weird.
>
> FWIW, a few observations, possibly relevant:
>
> (1) Each chunk of leaked output begins with 1 or 2 Unicode Replacement
> characters (U+FFFD). In the transcript I provided, the first leak
> began with 1 replacement char, and all later leaks began with 2.
>
> (2) The replacement characters are immediately followed by
> "@ build-log 30033 4096\n", and that string is also sprinkled
> throughout the leaked output, with approximately ~4060-4070
> characters of leaked output between each occurrence of
> "@ build-log 30033 4096\n".

Indeed. I managed to reproduce it while building modem-manager. I
strace’d ‘guix build’ with an additional ‘pk’¹ to see what happens, and
here’s what leads to the wrong “write(2, "�@ build-log…")” call:

Toggle snippet (30 lines)
read(13, "gmlo\0\0\0\0", 8) = 8
read(13, "\27\20\0\0\0\0\0\0", 8) = 8
read(13, "@ build-log 22090 4096\n […] warning: \342\200", 4119) = 4119
read(13, "\0", 1) = 1
write(1, "\n", 1) = 1
write(1, ";;; (write 1008 <> #f 0)\n", 25) = 25
write(1, "\n", 1) = 1
write(1, ";;; (write 985 <> 22090 4096)\n", 30) = 30
write(1, "\n", 1) = 1
write(1, ";;; (write 1008 <> 22090 3111)\n", 31) = 31
write(1, "\n", 1) = 1
write(1, ";;; (write 1008 <> 22090 2103)\n", 31) = 31
write(1, "\n", 1) = 1
write(1, ";;; (write 1008 <> 22090 1095)\n", 31) = 31
write(1, "\n", 1) = 1
write(1, ";;; (write 88 <> 22090 87)\n", 27) = 27
write(2, "\r\33[K\\ 'build' phase", 19) = 19
[…]
write(2, "\r\33[K\\ 'build' phase", 19) = 19
write(1, "\n", 1) = 1
write(1, ";;; (write 1 <> #f 0)\n", 22) = 22
read(13, "gmlo\0\0\0\0", 8) = 8
read(13, "\27\20\0\0\0\0\0\0", 8) = 8
read(13, "@ build-log 22090 4096\n\230g_simple_async_result_take_error\342\200\231 is deprecated[…]", 4119) = 4119
read(13, "\0", 1) = 1
write(1, "\n", 1) = 1
write(1, ";;; (write 1008 <> #f 0)\n", 25) = 25
write(2, "\357\277\275@ build-log 22090 4096\n", 26) = 26

The third read(2) call here ends on a partial UTF-8 sequence for LEFT
SINGLE QUOTATION MARK (we get the first two bytes of a three byte
sequence.)

What happens is that ‘process-stderr’ in (guix store) gets that byte
string from the daemon, passes it through ‘read-maybe-utf8-string’,
which replaces the last two bytes with REPLACEMENT CHARACTER, which is
itself a 3-byte sequence.

Thus, we have this extra byte that’s being inserted. That confuses the
whole machinery since the build log was announced as being 4096-byte
long, and it’s now 4097-byte long.

Internally, ‘build-event-output-port’ keeps the last byte of the
REPLACEMENT CHARACTER sequence in the ‘%fragments’ buffer.
Consequently, the “@ build-log” string that comes next doesn’t start on
a newline, and thus it is considered build output. Since the first byte
does not constitute a valid UTF-8 sequence, another REPLACEMENT
CHARACTER is inserted there when it gets printed.


So ‘build-event-output-port’ is working as expected. The problem is the
first layer of UTF-8 decoding that happens in ‘process-stderr’, in the
‘%stderr-next’ case. We would need to disable it, but only if the build
output port is ‘build-event-output-port’ (i.e., it’s capable of
interpreting “multiplexed build output” correctly.)

Thanks,
Ludo’.

¹ pk:
Toggle diff (12 lines)
diff --git a/guix/status.scm b/guix/status.scm
index cbea4151f2..4dcbcb0c1f 100644
--- a/guix/status.scm
+++ b/guix/status.scm
@@ -717,6 +717,7 @@ The second return value is a thunk to retrieve the current state."
(pointer->bytevector ptr count)))
(define (write! bv offset count)
+ (pk 'write count '<> %build-output-pid %build-output-left)
(if %build-output-pid
(let ((keep (min count %build-output-left)))
(set! %build-output
M
M
Mark H Weaver wrote on 26 Apr 2019 12:09
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
87k1fgh9c0.fsf@netris.org
Hi Ludovic,

Thanks for investigating this.

Ludovic Courtès <ludo@gnu.org> writes:

Toggle quote (9 lines)
> The third read(2) call here ends on a partial UTF-8 sequence for LEFT
> SINGLE QUOTATION MARK (we get the first two bytes of a three byte
> sequence.)
>
> What happens is that ‘process-stderr’ in (guix store) gets that byte
> string from the daemon, passes it through ‘read-maybe-utf8-string’,
> which replaces the last two bytes with REPLACEMENT CHARACTER, which is
> itself a 3-byte sequence.

It seems to me that what's needed here is to save the UTF-8 decoder
state between calls to 'process-stderr'. Coincidentally, I also needed
something like this a week ago, when I tried implementing R6RS custom
textual input/output ports on top of R6RS custom binary input/output
ports.

To meet these needs, I've implemented a fairly efficient, purely
functional UTF-8 decoder in Scheme that accepts a decoder state and an
arbitrary range from a bytevector, and returns a new decoder state.
There's a macro that allows arbitrary actions to be performed when a
code point (or maximal subpart in the case of errors) is found.

This macro is then used to implement a decoder (utf8->string!) that
writes into an arbitrary range of an existing string. Of course, it's
not purely functional, but it avoids heap allocation when compiled with
Guile. On my Thinkpad X200, it can process around 10 megabytes per
second.

The state is represented as an exact integer between 0 and #xF48FBF
inclusive, which are simply the bytes that have been seen so far in the
current code sequence, in big-endian order, or 0 for the start state.
For example, #xF48FBF represents the state where the bytes (F4 8F BF)
have been read. The state is always either 0 or a proper prefix of a
valid UTF-8 byte sequence.

I also plan to implement an optimized C version of 'utf8->string!' and
add it to Guile, in order to implement fast custom textual ports. The
precise name and API is not yet finalized. At present, 'utf8->string!'
always replaces maximal subparts with the substitution character in case
of errors, but I intend to eventually support other error modes as well.

What would you think about using this code to replace the uses of
'read-maybe-utf8-string', and storing the UTF-8 decoder state in the
<store-connection> object? Would we need to store multiple states in
case of (max-jobs > 1)?

Regards
Mark
Attachment: utf8-decoder.scm
M
M
Mark H Weaver wrote on 26 Apr 2019 17:45
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
87bm0sgts0.fsf@netris.org
Here's an improved version of the code with doc strings. It also
properly handles the case of (target-source >= target-end) in
'utf8->string!'.

Mark
Attachment: utf8-decoder.scm
M
M
Mark H Weaver wrote on 27 Apr 2019 00:56
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
877ebfhodh.fsf@netris.org
Here's version 3 with much more precise specifications in the
docstrings. If I recall correctly, the code itself is identical to
version 2.

Mark
Attachment: utf8-decoder.scm
L
L
Ludovic Courtès wrote on 27 Apr 2019 09:36
(name . Mark H Weaver)(address . mhw@netris.org)(address . 35350@debbugs.gnu.org)
874l6jh0bx.fsf@gnu.org
Hi Mark,

Mark H Weaver <mhw@netris.org> skribis:

Toggle quote (14 lines)
> Ludovic Courtès <ludo@gnu.org> writes:
>
>> The third read(2) call here ends on a partial UTF-8 sequence for LEFT
>> SINGLE QUOTATION MARK (we get the first two bytes of a three byte
>> sequence.)
>>
>> What happens is that ‘process-stderr’ in (guix store) gets that byte
>> string from the daemon, passes it through ‘read-maybe-utf8-string’,
>> which replaces the last two bytes with REPLACEMENT CHARACTER, which is
>> itself a 3-byte sequence.
>
> It seems to me that what's needed here is to save the UTF-8 decoder
> state between calls to 'process-stderr'.

So there are two things. To fix the issue you reported (build output
that goes through), I think we must simply turn off UTF-8 decoding from
‘process-stderr’ and leave that entirely to ‘build-event-output-port’.

However, ‘build-event-output-port’ would still fail to properly decode
split UTF-8 sequences, and for that we’d need to preserve decoder state
as you describe.

Toggle quote (23 lines)
> Coincidentally, I also needed something like this a week ago, when I
> tried implementing R6RS custom textual input/output ports on top of
> R6RS custom binary input/output ports.
>
> To meet these needs, I've implemented a fairly efficient, purely
> functional UTF-8 decoder in Scheme that accepts a decoder state and an
> arbitrary range from a bytevector, and returns a new decoder state.
> There's a macro that allows arbitrary actions to be performed when a
> code point (or maximal subpart in the case of errors) is found.
>
> This macro is then used to implement a decoder (utf8->string!) that
> writes into an arbitrary range of an existing string. Of course, it's
> not purely functional, but it avoids heap allocation when compiled with
> Guile. On my Thinkpad X200, it can process around 10 megabytes per
> second.
>
> The state is represented as an exact integer between 0 and #xF48FBF
> inclusive, which are simply the bytes that have been seen so far in the
> current code sequence, in big-endian order, or 0 for the start state.
> For example, #xF48FBF represents the state where the bytes (F4 8F BF)
> have been read. The state is always either 0 or a proper prefix of a
> valid UTF-8 byte sequence.

Awesome! I think that’s something we should definitely add to Guile
proper. We can use it in Guix before or after it’s included in Guile.

Thank you!

Ludo’.
M
M
Mark H Weaver wrote on 30 Apr 2019 13:26
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
87imuvme7g.fsf@netris.org
Hi Ludovic,

Ludovic Courtès <ludo@gnu.org> writes:

Toggle quote (20 lines)
> Mark H Weaver <mhw@netris.org> skribis:
>
>> Ludovic Courtès <ludo@gnu.org> writes:
>>
>>> The third read(2) call here ends on a partial UTF-8 sequence for LEFT
>>> SINGLE QUOTATION MARK (we get the first two bytes of a three byte
>>> sequence.)
>>>
>>> What happens is that ‘process-stderr’ in (guix store) gets that byte
>>> string from the daemon, passes it through ‘read-maybe-utf8-string’,
>>> which replaces the last two bytes with REPLACEMENT CHARACTER, which is
>>> itself a 3-byte sequence.
>>
>> It seems to me that what's needed here is to save the UTF-8 decoder
>> state between calls to 'process-stderr'.
>
> So there are two things. To fix the issue you reported (build output
> that goes through), I think we must simply turn off UTF-8 decoding from
> ‘process-stderr’ and leave that entirely to ‘build-event-output-port’.

Can we assume that UTF-8 is the appropriate encoding for
(current-build-output-port)? My interpretation of the Guix manual entry
for 'current-build-output-port' suggests that the answer should be "no".

Also, in your previous message you wrote:

The problem is the first layer of UTF-8 decoding that happens in
‘process-stderr’, in the ‘%stderr-next’ case. We would need to
disable it, but only if the build output port is
‘build-event-output-port’ (i.e., it’s capable of interpreting
“multiplexed build output” correctly.)

It sounds like you're suggesting that 'process-stderr' should look to
see if (current-build-output-port) is a 'build-event-output-port', and
in that case it should use binary I/O primitives to write raw binary
data to it, otherwise it should use text I/O primitives and write
characters to it. Do I understand correctly?

IMO, it would be cleaner to treat 'build-event-output-port' uniformly,
and specifically as a textual port of unknown encoding.

What do you think?

Toggle quote (4 lines)
> However, ‘build-event-output-port’ would still fail to properly decode
> split UTF-8 sequences, and for that we’d need to preserve decoder
> state as you describe.

I would suggest changing 'build-event-output-port' to create an R6RS
custom *textual* output port, so that it wouldn't have to worry about
encodings at all, and it would only be given whole characters.
Internally, it would be doing exactly what you suggest above, but those
details would be encapsulated within the custom textual port.

However, I don't think we can use Guile's current implementation of R6RS
custom textual output ports, which are currently built on Guile's legacy
soft ports, which I suspect have a similar bug with multibyte characters
sometimes being split (see 'soft_port_write' in vports.c).

Having said all of this, my suggestions would ultimately entail having
two separate places along the stderr pipeline where 'utf8->string!'
would be used, and maybe that's too much until we have a more optimized
C implementation of it.

Thoughts?

Mark
L
L
Ludovic Courtès wrote on 4 May 2019 02:33
(name . Mark H Weaver)(address . mhw@netris.org)(address . 35350@debbugs.gnu.org)
87r29e5zsw.fsf@gnu.org
Hi Mark,

Mark H Weaver <mhw@netris.org> skribis:

Toggle quote (2 lines)
> Ludovic Courtès <ludo@gnu.org> writes:

[...]

Toggle quote (8 lines)
>> So there are two things. To fix the issue you reported (build output
>> that goes through), I think we must simply turn off UTF-8 decoding from
>> ‘process-stderr’ and leave that entirely to ‘build-event-output-port’.
>
> Can we assume that UTF-8 is the appropriate encoding for
> (current-build-output-port)? My interpretation of the Guix manual entry
> for 'current-build-output-port' suggests that the answer should be "no".

What goes to ‘current-build-output-port’ comes from builds processes.
It’s usually UTF-8 but it can be anything, including binary garbage,
which should be gracefully handled.

That’s why ‘process-stderr’ currently uses ‘read-maybe-utf8-string’.

Toggle quote (14 lines)
> Also, in your previous message you wrote:
>
> The problem is the first layer of UTF-8 decoding that happens in
> ‘process-stderr’, in the ‘%stderr-next’ case. We would need to
> disable it, but only if the build output port is
> ‘build-event-output-port’ (i.e., it’s capable of interpreting
> “multiplexed build output” correctly.)
>
> It sounds like you're suggesting that 'process-stderr' should look to
> see if (current-build-output-port) is a 'build-event-output-port', and
> in that case it should use binary I/O primitives to write raw binary
> data to it, otherwise it should use text I/O primitives and write
> characters to it. Do I understand correctly?

Yes. (Actually, rather than guessing if (current-build-output-port) is
a ‘build-event-output-port’, there could be a fluid to ask for the use
of raw binary primitives.)

Toggle quote (3 lines)
> IMO, it would be cleaner to treat 'build-event-output-port' uniformly,
> and specifically as a textual port of unknown encoding.

(You mean ‘current-build-output-port’, right?)

I think you’re right. I’m not yet entirely sure what the implications
are. There’s a couple of tests in tests/store.scm for UTF-8
interpretation that describe behavior that I think we should preserve.

Toggle quote (16 lines)
> I would suggest changing 'build-event-output-port' to create an R6RS
> custom *textual* output port, so that it wouldn't have to worry about
> encodings at all, and it would only be given whole characters.
> Internally, it would be doing exactly what you suggest above, but those
> details would be encapsulated within the custom textual port.
>
> However, I don't think we can use Guile's current implementation of R6RS
> custom textual output ports, which are currently built on Guile's legacy
> soft ports, which I suspect have a similar bug with multibyte characters
> sometimes being split (see 'soft_port_write' in vports.c).
>
> Having said all of this, my suggestions would ultimately entail having
> two separate places along the stderr pipeline where 'utf8->string!'
> would be used, and maybe that's too much until we have a more optimized
> C implementation of it.

Yeah it looks like we don’t yet have custom textual output ports that we
could rely on, do we?

I support your work to add that in Guile proper!

Thanks,
Ludo’.
M
M
Mark H Weaver wrote on 4 May 2019 11:53
(name . Ludovic Courtès)(address . ludo@gnu.org)(address . 35350@debbugs.gnu.org)
87tveauk2u.fsf@netris.org
Hi Ludovic,

Ludovic Courtès <ludo@gnu.org> writes:

Toggle quote (20 lines)
> Mark H Weaver <mhw@netris.org> skribis:
>
>> Ludovic Courtès <ludo@gnu.org> writes:
>
> [...]
>
>>> So there are two things. To fix the issue you reported (build output
>>> that goes through), I think we must simply turn off UTF-8 decoding from
>>> ‘process-stderr’ and leave that entirely to ‘build-event-output-port’.
>>
>> Can we assume that UTF-8 is the appropriate encoding for
>> (current-build-output-port)? My interpretation of the Guix manual entry
>> for 'current-build-output-port' suggests that the answer should be "no".
>
> What goes to ‘current-build-output-port’ comes from builds processes.
> It’s usually UTF-8 but it can be anything, including binary garbage,
> which should be gracefully handled.
>
> That’s why ‘process-stderr’ currently uses ‘read-maybe-utf8-string’.

I agree that we should (permissively) interpret the build process output
as UTF-8, regardless of locale settings. However, the encoding of
'current-build-output-port' is orthogonal, and I see no reason to assume
that it's UTF-8.

As 'process-stderr' is currently implemented, it makes no assumptions
about the encoding of 'current-build-output-port'. That's because it
uses only textual I/O on it. The end result is that the UTF-8 build
output is effectively converted into the port encoding of
'current-build-output-port', whatever it might be. I think that's how
it should be, no?

Toggle quote (23 lines)
>> Also, in your previous message you wrote:
>>
>> The problem is the first layer of UTF-8 decoding that happens in
>> ‘process-stderr’, in the ‘%stderr-next’ case. We would need to
>> disable it, but only if the build output port is
>> ‘build-event-output-port’ (i.e., it’s capable of interpreting
>> “multiplexed build output” correctly.)
>>
>> It sounds like you're suggesting that 'process-stderr' should look to
>> see if (current-build-output-port) is a 'build-event-output-port', and
>> in that case it should use binary I/O primitives to write raw binary
>> data to it, otherwise it should use text I/O primitives and write
>> characters to it. Do I understand correctly?
>
> Yes. (Actually, rather than guessing if (current-build-output-port) is
> a ‘build-event-output-port’, there could be a fluid to ask for the use
> of raw binary primitives.)
>
>> IMO, it would be cleaner to treat 'build-event-output-port' uniformly,
>> and specifically as a textual port of unknown encoding.
>
> (You mean ‘current-build-output-port’, right?)

Yes, indeed.

Toggle quote (4 lines)
> I think you’re right. I’m not yet entirely sure what the implications
> are. There’s a couple of tests in tests/store.scm for UTF-8
> interpretation that describe behavior that I think we should preserve.

I certainly agree that we should preserve those tests. I would go
further and add two more tests that bind 'current-build-output-port' to
a port with a non-UTF-8 encoding (e.g. UTF-16) and verify that the λ
gets converted correctly. The test build process would output the λ as
UTF-8, but it should be written to 'current-build-output-port' as
e.g. UTF-16.

What do you think?

Toggle quote (21 lines)
>> I would suggest changing 'build-event-output-port' to create an R6RS
>> custom *textual* output port, so that it wouldn't have to worry about
>> encodings at all, and it would only be given whole characters.
>> Internally, it would be doing exactly what you suggest above, but those
>> details would be encapsulated within the custom textual port.
>>
>> However, I don't think we can use Guile's current implementation of R6RS
>> custom textual output ports, which are currently built on Guile's legacy
>> soft ports, which I suspect have a similar bug with multibyte characters
>> sometimes being split (see 'soft_port_write' in vports.c).
>>
>> Having said all of this, my suggestions would ultimately entail having
>> two separate places along the stderr pipeline where 'utf8->string!'
>> would be used, and maybe that's too much until we have a more optimized
>> C implementation of it.
>
> Yeah it looks like we don’t yet have custom textual output ports that we
> could rely on, do we?
>
> I support your work to add that in Guile proper!

For now, I can offer a new implementation of custom textual output ports
built upon custom binary ports and the 'utf8->string!' that I previously
sent. See attached.

Thanks,
Mark


Toggle snippet (32 lines)
GNU Guile 2.2.4
Copyright (C) 1995-2017 Free Software Foundation, Inc.

Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'.
This program is free software, and you are welcome to redistribute it
under certain conditions; type `,show c' for details.

Enter `,help' for help.
scheme@(guile-user)> (load "utf8-decoder.scm")
scheme@(guile-user)> (load "guile-new-custom-textual-ports.scm")
scheme@(guile-user)> (define (my-write! str start count)
(pk 'my-write! (substring str start (+ start count)))
count)
scheme@(guile-user)> (define port (make-custom-textual-output-port "test1" my-write! #f #f #f))
scheme@(guile-user)> (display "Hello λ world!" port)
scheme@(guile-user)> (force-output port)

;;; (my-write! "Hello λ world!")
scheme@(guile-user)> (string->utf8 "λ")
$2 = #vu8(206 187)
scheme@(guile-user)> (string->utf8 "Hello λ world!")
$3 = #vu8(72 101 108 108 111 32 206 187 32 119 111 114 108 100 33)
scheme@(guile-user)> (put-bytevector port #vu8(72 101 108 108 111 32 206))
scheme@(guile-user)> (force-output port)

;;; (my-write! "Hello ")
scheme@(guile-user)> (put-bytevector port #vu8(187 32 119 111 114 108 100 33))
scheme@(guile-user)> (force-output port)

;;; (my-write! "λ world!")
scheme@(guile-user)>
;;; Copyright © 2019 Mark H Weaver <mhw@netris.org> ;;; ;;; This program is free software: you can redistribute it and/or modify ;;; it under the terms of the GNU General Public License as published by ;;; the Free Software Foundation, either version 3 of the License, or ;;; (at your option) any later version. ;;; ;;; This program is distributed in the hope that it will be useful, ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;;; GNU General Public License for more details. ;;; ;;; You should have received a copy of the GNU General Public License ;;; along with this program. If not, see <http://www.gnu.org/licenses/>. (use-modules (rnrs io ports)) (define (make-custom-textual-output-port id write! get-position set-position! close) (let (;; Allocate a per-port string buffer which will be used as a ;; temporary buffer for decoding, to avoid heap allocation ;; during normal operation. (buffer (make-string 4096)) ;; 'state' is the UTF-8 decoder state, which represents a ;; proper prefix of a well-formed UTF-8 byte sequence. These ;; are bytes that 'binary-write!' has accepted and reported as ;; having been written, although we are not able to decode ;; them into a character to pass to (textual) 'write!' until ;; more bytes arrive. (state 0)) (define (binary-write! bv start count) (call-with-values (lambda () ;; XXX FIXME: Consider performing this ;; decoding strictly. (utf8->string! state bv start (+ start count) buffer 0 (string-length buffer))) (lambda (new-state bv-pos char-count) (let* (;; Avoid calling write! with (char-count = 0) unless ;; (count = 0) was passed to us, because calling ;; 'write!' with count=0 has a special meaning: it ;; means to pass an EOF object to the byte/character ;; sink. (chars-accepted (if (and (zero? char-count) (not (zero? count))) 0 (write! buffer 0 char-count))) ;; Compute 'bytes-accepted' in such a way that the ;; bytes from STATE are not included, because they ;; were passed to us in previous calls, and are not ;; part of the bytevector range that we are now being ;; asked to write. However, it's important to note ;; that if 'write!' did not accept the bytes from ;; STATE, 'bytes-accepted' will be negative. We must ;; handle that case specially below. (bytes-accepted (- count (string-utf8-length (substring buffer chars-accepted char-count))))) ;; If 'bytes-accepted' is negative, that means the bytes ;; from STATE were not written. This can only happen if ;; 'chars-accepted' is 0, because 'write!' can only accept ;; whole code points, and the bytes from STATE are part of ;; at most a single code point. In this case, we must ;; leave STATE unchanged and return 0. (if (negative? bytes-accepted) 0 (begin (set! state new-state) bytes-accepted)))))) (define (binary-close) (set! buffer #f) (when close (close))) (define port (make-custom-binary-output-port id binary-write! get-position set-position! binary-close)) ;; Always use UTF-8 as the encoding for custom textual ports, as ;; an internal implementation detail, to ensure that all Unicode ;; characters will pass through regardless of the current locale. (set-port-encoding! port "UTF-8") port))
L
L
Ludovic Courtès wrote on 9 Nov 2019 09:17
control message for bug #35350
(address . control@debbugs.gnu.org)
874kzdarzx.fsf@gnu.org
severity 35350 important
quit
S
S
Sarah Morgensen wrote on 19 Sep 2021 22:44
Re: bug#35350: Some compile output still leaks through with --verbosity=1
(name . Mark H Weaver)(address . mhw@netris.org)
864kafhkbs.fsf@mgsn.dev
Hello,

I encountered this issue today. This looks like a pretty complete
solution ready to go. Did this ever make it into Guile/Guix?

(Ironically I was also reaching for a "make-custom-textual-output-port"
the other day!)

--
Sarah

Mark H Weaver <mhw@netris.org> writes:

Toggle quote (220 lines)
> Hi Ludovic,
>
> Ludovic Courtès <ludo@gnu.org> writes:
>
>> Mark H Weaver <mhw@netris.org> skribis:
>>
>>> Ludovic Courtès <ludo@gnu.org> writes:
>>
>> [...]
>>
>>>> So there are two things. To fix the issue you reported (build output
>>>> that goes through), I think we must simply turn off UTF-8 decoding from
>>>> ‘process-stderr’ and leave that entirely to ‘build-event-output-port’.
>>>
>>> Can we assume that UTF-8 is the appropriate encoding for
>>> (current-build-output-port)? My interpretation of the Guix manual entry
>>> for 'current-build-output-port' suggests that the answer should be "no".
>>
>> What goes to ‘current-build-output-port’ comes from builds processes.
>> It’s usually UTF-8 but it can be anything, including binary garbage,
>> which should be gracefully handled.
>>
>> That’s why ‘process-stderr’ currently uses ‘read-maybe-utf8-string’.
>
> I agree that we should (permissively) interpret the build process output
> as UTF-8, regardless of locale settings. However, the encoding of
> 'current-build-output-port' is orthogonal, and I see no reason to assume
> that it's UTF-8.
>
> As 'process-stderr' is currently implemented, it makes no assumptions
> about the encoding of 'current-build-output-port'. That's because it
> uses only textual I/O on it. The end result is that the UTF-8 build
> output is effectively converted into the port encoding of
> 'current-build-output-port', whatever it might be. I think that's how
> it should be, no?
>
>>> Also, in your previous message you wrote:
>>>
>>> The problem is the first layer of UTF-8 decoding that happens in
>>> ‘process-stderr’, in the ‘%stderr-next’ case. We would need to
>>> disable it, but only if the build output port is
>>> ‘build-event-output-port’ (i.e., it’s capable of interpreting
>>> “multiplexed build output” correctly.)
>>>
>>> It sounds like you're suggesting that 'process-stderr' should look to
>>> see if (current-build-output-port) is a 'build-event-output-port', and
>>> in that case it should use binary I/O primitives to write raw binary
>>> data to it, otherwise it should use text I/O primitives and write
>>> characters to it. Do I understand correctly?
>>
>> Yes. (Actually, rather than guessing if (current-build-output-port) is
>> a ‘build-event-output-port’, there could be a fluid to ask for the use
>> of raw binary primitives.)
>>
>>> IMO, it would be cleaner to treat 'build-event-output-port' uniformly,
>>> and specifically as a textual port of unknown encoding.
>>
>> (You mean ‘current-build-output-port’, right?)
>
> Yes, indeed.
>
>> I think you’re right. I’m not yet entirely sure what the implications
>> are. There’s a couple of tests in tests/store.scm for UTF-8
>> interpretation that describe behavior that I think we should preserve.
>
> I certainly agree that we should preserve those tests. I would go
> further and add two more tests that bind 'current-build-output-port' to
> a port with a non-UTF-8 encoding (e.g. UTF-16) and verify that the λ
> gets converted correctly. The test build process would output the λ as
> UTF-8, but it should be written to 'current-build-output-port' as
> e.g. UTF-16.
>
> What do you think?
>
>>> I would suggest changing 'build-event-output-port' to create an R6RS
>>> custom *textual* output port, so that it wouldn't have to worry about
>>> encodings at all, and it would only be given whole characters.
>>> Internally, it would be doing exactly what you suggest above, but those
>>> details would be encapsulated within the custom textual port.
>>>
>>> However, I don't think we can use Guile's current implementation of R6RS
>>> custom textual output ports, which are currently built on Guile's legacy
>>> soft ports, which I suspect have a similar bug with multibyte characters
>>> sometimes being split (see 'soft_port_write' in vports.c).
>>>
>>> Having said all of this, my suggestions would ultimately entail having
>>> two separate places along the stderr pipeline where 'utf8->string!'
>>> would be used, and maybe that's too much until we have a more optimized
>>> C implementation of it.
>>
>> Yeah it looks like we don’t yet have custom textual output ports that we
>> could rely on, do we?
>>
>> I support your work to add that in Guile proper!
>
> For now, I can offer a new implementation of custom textual output ports
> built upon custom binary ports and the 'utf8->string!' that I previously
> sent. See attached.
>
> Thanks,
> Mark
>
> GNU Guile 2.2.4
> Copyright (C) 1995-2017 Free Software Foundation, Inc.
>
> Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'.
> This program is free software, and you are welcome to redistribute it
> under certain conditions; type `,show c' for details.
>
> Enter `,help' for help.
> scheme@(guile-user)> (load "utf8-decoder.scm")
> scheme@(guile-user)> (load "guile-new-custom-textual-ports.scm")
> scheme@(guile-user)> (define (my-write! str start count)
> (pk 'my-write! (substring str start (+ start count)))
> count)
> scheme@(guile-user)> (define port (make-custom-textual-output-port "test1" my-write! #f #f #f))
> scheme@(guile-user)> (display "Hello λ world!" port)
> scheme@(guile-user)> (force-output port)
>
> ;;; (my-write! "Hello λ world!")
> scheme@(guile-user)> (string->utf8 "λ")
> $2 = #vu8(206 187)
> scheme@(guile-user)> (string->utf8 "Hello λ world!")
> $3 = #vu8(72 101 108 108 111 32 206 187 32 119 111 114 108 100 33)
> scheme@(guile-user)> (put-bytevector port #vu8(72 101 108 108 111 32 206))
> scheme@(guile-user)> (force-output port)
>
> ;;; (my-write! "Hello ")
> scheme@(guile-user)> (put-bytevector port #vu8(187 32 119 111 114 108 100 33))
> scheme@(guile-user)> (force-output port)
>
> ;;; (my-write! "λ world!")
> scheme@(guile-user)>
>
> ;;; Copyright © 2019 Mark H Weaver <mhw@netris.org>
> ;;;
> ;;; This program is free software: you can redistribute it and/or modify
> ;;; it under the terms of the GNU General Public License as published by
> ;;; the Free Software Foundation, either version 3 of the License, or
> ;;; (at your option) any later version.
> ;;;
> ;;; This program is distributed in the hope that it will be useful,
> ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
> ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> ;;; GNU General Public License for more details.
> ;;;
> ;;; You should have received a copy of the GNU General Public License
> ;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
>
> (use-modules (rnrs io ports))
>
> (define (make-custom-textual-output-port id
> write!
> get-position
> set-position!
> close)
> (let (;; Allocate a per-port string buffer which will be used as a
> ;; temporary buffer for decoding, to avoid heap allocation
> ;; during normal operation.
> (buffer (make-string 4096))
> ;; 'state' is the UTF-8 decoder state, which represents a
> ;; proper prefix of a well-formed UTF-8 byte sequence. These
> ;; are bytes that 'binary-write!' has accepted and reported as
> ;; having been written, although we are not able to decode
> ;; them into a character to pass to (textual) 'write!' until
> ;; more bytes arrive.
> (state 0))
> (define (binary-write! bv start count)
> (call-with-values (lambda ()
> ;; XXX FIXME: Consider performing this
> ;; decoding strictly.
> (utf8->string! state bv start (+ start count)
> buffer 0 (string-length buffer)))
> (lambda (new-state bv-pos char-count)
> (let* (;; Avoid calling write! with (char-count = 0) unless
> ;; (count = 0) was passed to us, because calling
> ;; 'write!' with count=0 has a special meaning: it
> ;; means to pass an EOF object to the byte/character
> ;; sink.
> (chars-accepted (if (and (zero? char-count)
> (not (zero? count)))
> 0
> (write! buffer 0 char-count)))
> ;; Compute 'bytes-accepted' in such a way that the
> ;; bytes from STATE are not included, because they
> ;; were passed to us in previous calls, and are not
> ;; part of the bytevector range that we are now being
> ;; asked to write. However, it's important to note
> ;; that if 'write!' did not accept the bytes from
> ;; STATE, 'bytes-accepted' will be negative. We must
> ;; handle that case specially below.
> (bytes-accepted (- count (string-utf8-length
> (substring buffer
> chars-accepted
> char-count)))))
> ;; If 'bytes-accepted' is negative, that means the bytes
> ;; from STATE were not written. This can only happen if
> ;; 'chars-accepted' is 0, because 'write!' can only accept
> ;; whole code points, and the bytes from STATE are part of
> ;; at most a single code point. In this case, we must
> ;; leave STATE unchanged and return 0.
> (if (negative? bytes-accepted)
> 0
> (begin
> (set! state new-state)
> bytes-accepted))))))
> (define (binary-close)
> (set! buffer #f)
> (when close (close)))
> (define port
> (make-custom-binary-output-port id
> binary-write!
> get-position
> set-position!
> binary-close))
> ;; Always use UTF-8 as the encoding for custom textual ports, as
> ;; an internal implementation detail, to ensure that all Unicode
> ;; characters will pass through regardless of the current locale.
> (set-port-encoding! port "UTF-8")
> port))
?