[ Mar 20 21:50:45 Enabled. ]
[ Mar 20 21:50:45 Rereading configuration. ]
[ Mar 20 21:50:50 Rereading configuration. ]
[ Mar 20 21:50:57 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:50:57.193317975Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:50:57.196375597Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:50:57.196385187Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:50:57.215388085Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:50:57.30529457Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.281775193Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:49854"}
{"msg":"region db5800d4-4b68-44f5-8a90-acf697365b22 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.30110759Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.304701016Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions","method":"POST","req_id":"39ae5533-1ddc-418e-aaca-0cda72e130b1","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":10479,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.304731097Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible/regions/db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.314861088Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible/regions/db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.31493879Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"db5800d4-4b68-44f5-8a90-acf697365b22\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/db5800d4-4b68-44f5-8a90-acf697365b22\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.405207985Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"db5800d4-4b68-44f5-8a90-acf697365b22","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"db5800d4-4b68-44f5-8a90-acf697365b22\" \"--data\" \"/data/regions/db5800d4-4b68-44f5-8a90-acf697365b22\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.407224179Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"db5800d4-4b68-44f5-8a90-acf697365b22","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.558424322Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"db5800d4-4b68-44f5-8a90-acf697365b22","component":"worker"}
{"msg":"region db5800d4-4b68-44f5-8a90-acf697365b22 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.558532625Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.558844214Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.558898355Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions","method":"POST","req_id":"fdc8339d-6b5b-417a-8b45-8d2b6e8edb84","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":516,"response_code":"200"}
{"msg":"creating missing downstairs instance downstairs-db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.5594592Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.567747712Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.584760677Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.590158781Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.591734713Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.591810926Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.591860887Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.591897628Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.598245787Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:15.607133445Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.386554663Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions/db5800d4-4b68-44f5-8a90-acf697365b22","method":"GET","req_id":"71806f0b-4711-4f17-aabc-51ea9beda947","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/db5800d4-4b68-44f5-8a90-acf697365b22\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.388869905Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"dataset is oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible/regions/db5800d4-4b68-44f5-8a90-acf697365b22","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.401582715Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"region db5800d4-4b68-44f5-8a90-acf697365b22 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.41261436Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.412793315Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.412858237Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions/db5800d4-4b68-44f5-8a90-acf697365b22","method":"DELETE","req_id":"49dc778c-a1c1-495e-904f-8acbadda8dbb","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24028,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-db5800d4-4b68-44f5-8a90-acf697365b22 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.413941205Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.416781061Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions/db5800d4-4b68-44f5-8a90-acf697365b22","method":"GET","req_id":"a0566ea8-1e33-498a-8558-d25380c28a9d","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.420265875Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible/regions/db5800d4-4b68-44f5-8a90-acf697365b22\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.434639709Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"db5800d4-4b68-44f5-8a90-acf697365b22","component":"worker"}
{"msg":"region db5800d4-4b68-44f5-8a90-acf697365b22 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.519598831Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:16.524087611Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions/db5800d4-4b68-44f5-8a90-acf697365b22","method":"GET","req_id":"9dacecb1-52d2-4bc7-adfc-05e5519aa018","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"region 2d283ab4-ee1d-49c5-ba3c-195630fb05d7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.46965703Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.469818954Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions","method":"POST","req_id":"20c6c7b7-2237-45b6-a4a8-01e9c1336b3d","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":213,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.469844355Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible/regions/2d283ab4-ee1d-49c5-ba3c-195630fb05d7","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.478937018Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a7cc222c-73d4-4d3a-9803-c2ce9572803a/crucible/regions/2d283ab4-ee1d-49c5-ba3c-195630fb05d7","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.478974379Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2d283ab4-ee1d-49c5-ba3c-195630fb05d7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2d283ab4-ee1d-49c5-ba3c-195630fb05d7\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.544224174Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"2d283ab4-ee1d-49c5-ba3c-195630fb05d7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2d283ab4-ee1d-49c5-ba3c-195630fb05d7\" \"--data\" \"/data/regions/2d283ab4-ee1d-49c5-ba3c-195630fb05d7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.544286616Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"2d283ab4-ee1d-49c5-ba3c-195630fb05d7","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.639683647Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"region":"2d283ab4-ee1d-49c5-ba3c-195630fb05d7","component":"worker"}
{"msg":"region 2d283ab4-ee1d-49c5-ba3c-195630fb05d7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.639735738Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.639813351Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2d283ab4-ee1d-49c5-ba3c-195630fb05d7","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.641258729Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2d283ab4-ee1d-49c5-ba3c-195630fb05d7","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.644721252Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.652906311Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2d283ab4-ee1d-49c5-ba3c-195630fb05d7","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.656955589Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2d283ab4-ee1d-49c5-ba3c-195630fb05d7","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.657263447Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.657328399Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.65737486Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.657429832Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.661581913Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.665983931Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-20T21:51:18.693078445Z","hostname":"oxz_crucible_544ee685-d79b-455d-ba94-c9aceb71ee5e","pid":24423,"uri":"/crucible/0/regions","method":"POST","req_id":"92ea04bb-7cbc-4a4a-be88-ec6892ef793f","remote_addr":"[fd00:1122:3344:101::b]:49854","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":"200"}