[ Mar  3 23:31:14 Enabled. ]
[ Mar  3 23:31:14 Rereading configuration. ]
[ Mar  3 23:31:16 Rereading configuration. ]
[ Mar  3 23:31:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d984dabe-878c-4fe6-a385-4fd19a2c7ee9/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d984dabe-878c-4fe6-a385-4fd19a2c7ee9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.015258196Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.018651973Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.018660263Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.035795272Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:31:21.314291371Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.696893566Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35458"}
{"msg":"region 98fbbd3d-ecaa-47be-8207-2c7de2a4225b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.735019054Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.738210596Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions","method":"POST","req_id":"2747f568-a4c7-4f5c-b9c7-11dda0f17ad8","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17397,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.738233376Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_d984dabe-878c-4fe6-a385-4fd19a2c7ee9/crucible/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.746900034Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d984dabe-878c-4fe6-a385-4fd19a2c7ee9/crucible/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.746927905Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"98fbbd3d-ecaa-47be-8207-2c7de2a4225b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.802020997Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"region":"98fbbd3d-ecaa-47be-8207-2c7de2a4225b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"98fbbd3d-ecaa-47be-8207-2c7de2a4225b\" \"--data\" \"/data/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.804069336Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"region":"98fbbd3d-ecaa-47be-8207-2c7de2a4225b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.94996713Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions","method":"POST","req_id":"d583d346-3fb9-45c9-8bc6-468b47b8ec96","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.950078383Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"region":"98fbbd3d-ecaa-47be-8207-2c7de2a4225b","component":"worker"}
{"msg":"region 98fbbd3d-ecaa-47be-8207-2c7de2a4225b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.950096674Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.950280919Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.950741072Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.958261277Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.969737494Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.9727265Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.973465481Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.973535823Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.973577644Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.973623795Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.979506783Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:04.993788081Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:05.594655792Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions","method":"POST","req_id":"dfb524ab-1537-4482-a097-cb11466bb568","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.012545791Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","method":"GET","req_id":"49051566-6759-4e95-a1c1-7f11ffce7243","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2033,"response_code":"200"}
{"msg":"path is \"/data/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.017811201Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"dataset is oxp_d984dabe-878c-4fe6-a385-4fd19a2c7ee9/crucible/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.031144121Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"region 98fbbd3d-ecaa-47be-8207-2c7de2a4225b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.043283797Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.043441062Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.043524034Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","method":"DELETE","req_id":"e1fa1d4a-61ec-46af-8fd7-b982d43db18b","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25757,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-98fbbd3d-ecaa-47be-8207-2c7de2a4225b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.044665867Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.049401542Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","method":"GET","req_id":"b72f9e75-2cdf-4531-b961-a2e1b0faf46b","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.052623904Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d984dabe-878c-4fe6-a385-4fd19a2c7ee9/crucible/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.065897543Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"region":"98fbbd3d-ecaa-47be-8207-2c7de2a4225b","component":"worker"}
{"msg":"region 98fbbd3d-ecaa-47be-8207-2c7de2a4225b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.150739145Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T23:32:06.165862236Z","hostname":"oxz_crucible_feb91c17-00ef-4b29-8ec2-deb92fe6fa51","pid":23654,"uri":"/crucible/0/regions/98fbbd3d-ecaa-47be-8207-2c7de2a4225b","method":"GET","req_id":"7d63357c-66b5-43f6-ac6c-028d684ea4ae","remote_addr":"[fd00:1122:3344:101::a]:35458","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":"200"}