[ Mar 21 02:04:11 Enabled. ]
[ Mar 21 02:04:11 Rereading configuration. ]
[ Mar 21 02:04:13 Rereading configuration. ]
[ Mar 21 02:04:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_94750eb0-e711-4f25-8449-a0383aa1871a/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_94750eb0-e711-4f25-8449-a0383aa1871a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:18.324999418Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:18.328144383Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:18.328157323Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:18.34862872Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:18.454273079Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.456273391Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:43017"}
{"msg":"region e91abbcb-3930-49b2-84b4-5df72e42f8f9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.489040651Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.491799209Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions","method":"POST","req_id":"db375e4d-098a-48d4-bc07-ed9ed2e81076","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":11290,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.491827599Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_94750eb0-e711-4f25-8449-a0383aa1871a/crucible/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.502145769Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_94750eb0-e711-4f25-8449-a0383aa1871a/crucible/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.50221318Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e91abbcb-3930-49b2-84b4-5df72e42f8f9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.590120151Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"region":"e91abbcb-3930-49b2-84b4-5df72e42f8f9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e91abbcb-3930-49b2-84b4-5df72e42f8f9\" \"--data\" \"/data/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.590208222Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"region":"e91abbcb-3930-49b2-84b4-5df72e42f8f9","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.85147124Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions","method":"POST","req_id":"1fb4142d-3ce0-4df2-949c-36d5b903b04d","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.928229117Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"region":"e91abbcb-3930-49b2-84b4-5df72e42f8f9","component":"worker"}
{"msg":"region e91abbcb-3930-49b2-84b4-5df72e42f8f9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.928302578Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.928557733Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.92901997Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.943498223Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.969342992Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.980907774Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.981652507Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.981719558Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.981760319Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.981804569Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:36.990842607Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.016123127Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.206769706Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions","method":"POST","req_id":"00e7d38b-c2a1-4559-9381-8e2b66ac253b","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.817961996Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","method":"GET","req_id":"f5cf6b1b-6126-484a-b7d5-2ddeb70ec551","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2367,"response_code":"200"}
{"msg":"path is \"/data/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.821025879Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"dataset is oxp_94750eb0-e711-4f25-8449-a0383aa1871a/crucible/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.832747343Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"region e91abbcb-3930-49b2-84b4-5df72e42f8f9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.842841379Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.843097623Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.843142894Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","method":"DELETE","req_id":"edf80d71-a7c2-4a69-a578-2e3a7b46aefe","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22155,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e91abbcb-3930-49b2-84b4-5df72e42f8f9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.844205833Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.846673946Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","method":"GET","req_id":"4f144f4c-a05c-4218-b59e-d82ef2fc93e6","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.852431756Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_94750eb0-e711-4f25-8449-a0383aa1871a/crucible/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.863938686Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"region":"e91abbcb-3930-49b2-84b4-5df72e42f8f9","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.936600401Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","method":"GET","req_id":"86e8f9ab-1070-4e32-8d88-df3b1f4fcaf9","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region e91abbcb-3930-49b2-84b4-5df72e42f8f9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:37.965450624Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-21T02:04:38.065881772Z","hostname":"oxz_crucible_23db583e-ccb8-4fa4-9c20-a54f1e3bff09","pid":22793,"uri":"/crucible/0/regions/e91abbcb-3930-49b2-84b4-5df72e42f8f9","method":"GET","req_id":"1a0a6fe3-049f-41b2-b9be-891a3d1e76cb","remote_addr":"[fd00:1122:3344:101::b]:43017","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":42,"response_code":"200"}