[ Oct 10 04:25:16 Enabled. ]
[ Oct 10 04:25:16 Rereading configuration. ]
[ Oct 10 04:25:17 Rereading configuration. ]
[ Oct 10 04:25:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1738345c-a151-4031-b61e-f7f6cb49d7b6/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1738345c-a151-4031-b61e-f7f6cb49d7b6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:22.44007782Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:22.441325784Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:22.441334214Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:22.456023937Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:22.671393716Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.108981661Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60648"}
{"msg":"region fc9c0004-c943-40ab-aaa2-e84b4a29092f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.146851229Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.151873817Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions","method":"POST","req_id":"9d7e26c4-7f6b-44c5-a83f-60470bbf228b","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":14489,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.153038849Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_1738345c-a151-4031-b61e-f7f6cb49d7b6/crucible/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.162429197Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_1738345c-a151-4031-b61e-f7f6cb49d7b6/crucible/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.162468068Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fc9c0004-c943-40ab-aaa2-e84b4a29092f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.216930883Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"region":"fc9c0004-c943-40ab-aaa2-e84b4a29092f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fc9c0004-c943-40ab-aaa2-e84b4a29092f\" \"--data\" \"/data/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.221971861Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"region":"fc9c0004-c943-40ab-aaa2-e84b4a29092f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.308808404Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions","method":"POST","req_id":"ae7da2dc-1350-4cf1-b911-4667861e5e84","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.344741309Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"region":"fc9c0004-c943-40ab-aaa2-e84b4a29092f","component":"worker"}
{"msg":"region fc9c0004-c943-40ab-aaa2-e84b4a29092f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.344782811Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.344891863Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.347505665Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.352238115Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.37100546Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.373881189Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.381366624Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.381433486Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.381477667Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.381535709Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.385530039Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.390542446Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:54.590932734Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions","method":"POST","req_id":"0888d176-3e5f-41c8-9f63-6c477e5abff6","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.23182548Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","method":"GET","req_id":"9e0486b4-d0b9-418f-9d3f-b79338ffb70a","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.234609926Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"dataset is oxp_1738345c-a151-4031-b61e-f7f6cb49d7b6/crucible/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.246611155Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"region fc9c0004-c943-40ab-aaa2-e84b4a29092f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.256719693Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.256865697Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.256955169Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","method":"DELETE","req_id":"90de126d-0efc-40a5-9193-07381c18bcac","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22378,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-fc9c0004-c943-40ab-aaa2-e84b4a29092f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.258028119Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.259128439Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","method":"GET","req_id":"5ea7127b-246f-4d5f-b57b-acaca6f7e5de","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":41,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.263734405Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1738345c-a151-4031-b61e-f7f6cb49d7b6/crucible/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.276373732Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"region":"fc9c0004-c943-40ab-aaa2-e84b4a29092f","component":"worker"}
{"msg":"region fc9c0004-c943-40ab-aaa2-e84b4a29092f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.342136146Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-10T04:25:55.400086776Z","hostname":"oxz_crucible_9eefc15b-8608-4452-aeb0-514c08fa5527","pid":22984,"uri":"/crucible/0/regions/fc9c0004-c943-40ab-aaa2-e84b4a29092f","method":"GET","req_id":"d58ed542-40b7-44d2-8924-2dc54f432d4b","remote_addr":"[fd00:1122:3344:101::a]:60648","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":671,"response_code":"200"}