[ May 29 21:50:15 Enabled. ]
[ May 29 21:50:15 Rereading configuration. ]
[ May 29 21:50:17 Rereading configuration. ]
[ May 29 21:50:19 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.009394504Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.013722518Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.013743458Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.041671966Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.178237684Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.683827706Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52262"}
{"msg":"region 2134ad3a-815e-411a-ad06-f0b49a22a048 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.69458266Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.696554955Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"uri":"/crucible/0/regions","method":"POST","req_id":"f2472c06-b94a-48f6-822d-bb182d116d4d","remote_addr":"[fd00:1122:3344:101::a]:52262","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":6968,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.696574325Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.707331959Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.707391059Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2134ad3a-815e-411a-ad06-f0b49a22a048\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2134ad3a-815e-411a-ad06-f0b49a22a048\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.796129852Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"region":"2134ad3a-815e-411a-ad06-f0b49a22a048","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2134ad3a-815e-411a-ad06-f0b49a22a048\" \"--data\" \"/data/regions/2134ad3a-815e-411a-ad06-f0b49a22a048\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.798244078Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"region":"2134ad3a-815e-411a-ad06-f0b49a22a048","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.964174501Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"region":"2134ad3a-815e-411a-ad06-f0b49a22a048","component":"worker"}
{"msg":"region 2134ad3a-815e-411a-ad06-f0b49a22a048 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.964271403Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.964467515Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.96489892Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.967541993Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"uri":"/crucible/0/regions","method":"POST","req_id":"cbc12492-1f2e-4a3b-9bed-8f79e212f6fd","remote_addr":"[fd00:1122:3344:101::a]:52262","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.968376674Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.977632889Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.981067732Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.98178749Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.981905312Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.981974053Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.981985813Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.9866092Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.991996197Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.346247633Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41307"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.349525214Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"uri":"/crucible/0/regions/2134ad3a-815e-411a-ad06-f0b49a22a048","method":"GET","req_id":"6e4e3204-aa7f-41d0-a980-8169a10610ad","remote_addr":"[fd00:1122:3344:101::a]:41307","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3249,"response_code":"200"}
{"msg":"path is \"/data/regions/2134ad3a-815e-411a-ad06-f0b49a22a048\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.349803837Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"dataset is oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.358659747Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"snapshot is oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048@61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.366184861Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048\", \"61a6668b-d30e-4ad8-b5b4-2668f0f727ec\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.366214071Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.379453876Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"uri":"/crucible/0/regions/2134ad3a-815e-411a-ad06-f0b49a22a048/snapshots/61a6668b-d30e-4ad8-b5b4-2668f0f727ec","method":"GET","req_id":"3a55850a-4ab6-413a-9ea3-be0c2d8d045e","remote_addr":"[fd00:1122:3344:101::a]:41307","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29684,"response_code":"200"}
{"msg":"path is \"/data/regions/2134ad3a-815e-411a-ad06-f0b49a22a048\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.379632488Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"dataset is oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.388318236Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"snapshot is oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048@61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.397128446Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crucible/regions/2134ad3a-815e-411a-ad06-f0b49a22a048\", \"61a6668b-d30e-4ad8-b5b4-2668f0f727ec\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.397187246Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2134ad3a-815e-411a-ad06-f0b49a22a048-61a6668b-d30e-4ad8-b5b4-2668f0f727ec state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.406148648Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}
{"msg":"applying SMF actions for region 2134ad3a-815e-411a-ad06-f0b49a22a048 running snapshot 61a6668b-d30e-4ad8-b5b4-2668f0f727ec (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.40712149Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.407159341Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"uri":"/crucible/0/regions/2134ad3a-815e-411a-ad06-f0b49a22a048/snapshots/61a6668b-d30e-4ad8-b5b4-2668f0f727ec/run","method":"POST","req_id":"4dba4145-f42c-4bae-8481-c778b116389a","remote_addr":"[fd00:1122:3344:101::a]:41307","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27560,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-2134ad3a-815e-411a-ad06-f0b49a22a048-61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.415672696Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2134ad3a-815e-411a-ad06-f0b49a22a048-61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.422591412Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2134ad3a-815e-411a-ad06-f0b49a22a048-61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.434350899Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2134ad3a-815e-411a-ad06-f0b49a22a048/.zfs/snapshot/61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.434653222Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.434703623Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.434744113Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.434780214Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.434827905Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.43769912Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.441920903Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"worker"}
{"msg":"region 2134ad3a-815e-411a-ad06-f0b49a22a048 running snapshot 61a6668b-d30e-4ad8-b5b4-2668f0f727ec state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.441938543Z","hostname":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","pid":23327,"component":"datafile"}