[ Aug 16 07:16:36 Enabled. ]
[ Aug 16 07:16:36 Rereading configuration. ]
[ Aug 16 07:16:38 Rereading configuration. ]
[ Aug 16 07:16:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.028920284Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.030190653Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.030203384Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.062286145Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:41.274473905Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.762578321Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64462"}
{"msg":"region 124a1644-8d10-4218-959f-2b9a64fe0b64 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.764219709Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.765098739Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.765114529Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"d50bcb9a-2361-4ddf-bb4c-e61541ed8631","remote_addr":"[fd00:1122:3344:101::b]:64462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1104,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.775315165Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.775354246Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"124a1644-8d10-4218-959f-2b9a64fe0b64\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/124a1644-8d10-4218-959f-2b9a64fe0b64\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.847673826Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"124a1644-8d10-4218-959f-2b9a64fe0b64","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"124a1644-8d10-4218-959f-2b9a64fe0b64\" \"--data\" \"/data/regions/124a1644-8d10-4218-959f-2b9a64fe0b64\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.849606871Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"124a1644-8d10-4218-959f-2b9a64fe0b64","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.968035006Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"124a1644-8d10-4218-959f-2b9a64fe0b64","component":"worker"}
{"msg":"region 124a1644-8d10-4218-959f-2b9a64fe0b64 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.968083167Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.968180019Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.968620169Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.973971733Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.983639747Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.989705887Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.990578097Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.990680549Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.9907389Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.990791602Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:58.996919223Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.001870108Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.030284534Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"bab27a12-e6e6-45e8-8400-3240ab256872","remote_addr":"[fd00:1122:3344:101::b]:64462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":117,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.775230648Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","method":"GET","req_id":"58f656a3-279a-47cd-8657-1a71a07f39c0","remote_addr":"[fd00:1122:3344:101::b]:64462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/124a1644-8d10-4218-959f-2b9a64fe0b64\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.776985258Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"dataset is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.785627388Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"region 124a1644-8d10-4218-959f-2b9a64fe0b64 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.793567791Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.793699554Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.793758795Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","method":"DELETE","req_id":"3cd4a51b-8ef3-45b1-baa7-67d03749836a","remote_addr":"[fd00:1122:3344:101::b]:64462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":16793,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-124a1644-8d10-4218-959f-2b9a64fe0b64 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.79482291Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.795555107Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","method":"GET","req_id":"54c90831-fda0-4bce-8072-b9d2b6444e2a","remote_addr":"[fd00:1122:3344:101::b]:64462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.799356095Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/124a1644-8d10-4218-959f-2b9a64fe0b64\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.809273874Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"124a1644-8d10-4218-959f-2b9a64fe0b64","component":"worker"}
{"msg":"region 124a1644-8d10-4218-959f-2b9a64fe0b64 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.86108911Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:16:59.947974057Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/124a1644-8d10-4218-959f-2b9a64fe0b64","method":"GET","req_id":"ebbb0c9e-d796-4198-8060-caa096631328","remote_addr":"[fd00:1122:3344:101::b]:64462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.883699739Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54855"}
{"msg":"region 110d67ad-dd99-4995-aee3-88367ce9c7d6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.883830822Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.883989635Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.884004816Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"63cbb44b-75f6-4de1-9995-52f703df3e0d","remote_addr":"[fd00:1122:3344:101::a]:54855","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":201,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.892598054Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:54.892635805Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.117489548Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"2a8ffd8c-806f-4a84-9cf7-b88db461ea90","remote_addr":"[fd00:1122:3344:101::a]:54855","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"creating region Region { id: RegionId(\"110d67ad-dd99-4995-aee3-88367ce9c7d6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.17687386Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"110d67ad-dd99-4995-aee3-88367ce9c7d6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"110d67ad-dd99-4995-aee3-88367ce9c7d6\" \"--data\" \"/data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.176934921Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"110d67ad-dd99-4995-aee3-88367ce9c7d6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.679881426Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"10697b26-babb-4bd6-a796-f5b736c09c51","remote_addr":"[fd00:1122:3344:101::a]:54855","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.749237068Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"110d67ad-dd99-4995-aee3-88367ce9c7d6","component":"worker"}
{"msg":"region 110d67ad-dd99-4995-aee3-88367ce9c7d6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.749268499Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.749468154Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.751140712Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.768897122Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.855546934Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.883739425Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.884031161Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.884086883Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.884131254Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.884171885Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.924835174Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:55.961577182Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:17:57.13141156Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"8e84b28e-b9df-49f4-8f29-e65ff413bb9e","remote_addr":"[fd00:1122:3344:101::a]:54855","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.243604076Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54184"}
{"msg":"region caefb498-47b5-4230-91d4-bfea0189930a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.244434355Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.245569432Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.245587872Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"d8dbdc12-1efc-4a98-9df4-0a3a3d1ee6b5","remote_addr":"[fd00:1122:3344:101::a]:54184","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":1746,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/caefb498-47b5-4230-91d4-bfea0189930a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.258408178Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/caefb498-47b5-4230-91d4-bfea0189930a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.258439499Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"caefb498-47b5-4230-91d4-bfea0189930a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/caefb498-47b5-4230-91d4-bfea0189930a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.31824578Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"caefb498-47b5-4230-91d4-bfea0189930a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"caefb498-47b5-4230-91d4-bfea0189930a\" \"--data\" \"/data/regions/caefb498-47b5-4230-91d4-bfea0189930a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.320329748Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"caefb498-47b5-4230-91d4-bfea0189930a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.393064658Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"region":"caefb498-47b5-4230-91d4-bfea0189930a","component":"worker"}
{"msg":"region caefb498-47b5-4230-91d4-bfea0189930a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.393111389Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.393221171Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-caefb498-47b5-4230-91d4-bfea0189930a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.399796534Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-caefb498-47b5-4230-91d4-bfea0189930a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.40787797Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.415904735Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-caefb498-47b5-4230-91d4-bfea0189930a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.418779292Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/caefb498-47b5-4230-91d4-bfea0189930a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.419079979Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.41913187Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.419179841Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.419221632Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.42304517Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.427658697Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:31.497774395Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions","method":"POST","req_id":"bc8ee19f-7f1e-4c34-88ca-dbde89bd3d0e","remote_addr":"[fd00:1122:3344:101::a]:54184","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.88141224Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","method":"GET","req_id":"b3f1aa80-8f7a-43d2-b8ee-3a05cf8159ad","remote_addr":"[fd00:1122:3344:101::a]:54184","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":47,"response_code":"200"}
{"msg":"path is \"/data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.888068454Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"dataset is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.896786406Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"snapshot is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6@457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.904103544Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\", \"457023dd-1cbf-42e0-adb9-7dc2029f6007\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.904135865Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.914811332Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6/snapshots/457023dd-1cbf-42e0-adb9-7dc2029f6007","method":"GET","req_id":"cb0c2de5-8b24-49c6-996b-291347178bdc","remote_addr":"[fd00:1122:3344:101::a]:54184","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":30692,"response_code":"200"}
{"msg":"path is \"/data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.917718809Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"dataset is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.92513885Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"snapshot is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6@457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.932946791Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\", \"457023dd-1cbf-42e0-adb9-7dc2029f6007\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.932980971Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 110d67ad-dd99-4995-aee3-88367ce9c7d6-457023dd-1cbf-42e0-adb9-7dc2029f6007 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.942521842Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"applying SMF actions for region 110d67ad-dd99-4995-aee3-88367ce9c7d6 running snapshot 457023dd-1cbf-42e0-adb9-7dc2029f6007 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.942690166Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.942737827Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6/snapshots/457023dd-1cbf-42e0-adb9-7dc2029f6007/run","method":"POST","req_id":"49db564b-1750-4258-831e-54418846d942","remote_addr":"[fd00:1122:3344:101::a]:54184","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25049,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-110d67ad-dd99-4995-aee3-88367ce9c7d6-457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.945310546Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-110d67ad-dd99-4995-aee3-88367ce9c7d6-457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.949507163Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-110d67ad-dd99-4995-aee3-88367ce9c7d6-457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.96063524Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6/.zfs/snapshot/457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.960909087Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.960967108Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.961007809Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.96104763Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.961104781Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.96541792Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.970475877Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"worker"}
{"msg":"region 110d67ad-dd99-4995-aee3-88367ce9c7d6 running snapshot 457023dd-1cbf-42e0-adb9-7dc2029f6007 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:32.970488448Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"path is \"/data/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:33.2191171Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"dataset is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:33.227807171Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"datafile"}
{"msg":"snapshot is oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6@457023dd-1cbf-42e0-adb9-7dc2029f6007","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:33.235971409Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_32dc67d8-c3d1-40d3-a330-91e503ec31dd/crucible/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6\", \"457023dd-1cbf-42e0-adb9-7dc2029f6007\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:33.23600798Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T07:23:33.243659287Z","hostname":"oxz_crucible_ac3575fe-e30b-4ddb-b942-e1a3a0a89ea9","pid":25543,"uri":"/crucible/0/regions/110d67ad-dd99-4995-aee3-88367ce9c7d6/snapshots/457023dd-1cbf-42e0-adb9-7dc2029f6007/run","method":"POST","req_id":"dcf16c65-a0c0-4af8-8b5c-a6b62c762241","remote_addr":"[fd00:1122:3344:101::a]:54184","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25203,"response_code":"200"}