[ Jun 18 18:35:35 Enabled. ]
[ Jun 18 18:35:35 Rereading configuration. ]
[ Jun 18 18:35:37 Rereading configuration. ]
[ Jun 18 18:35:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:35:40.62533711Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:35:40.628874858Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:35:40.628884759Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:35:40.649733486Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:35:40.71672485Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.839705352Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37445"}
{"msg":"region 30104d9b-d8b7-4337-948d-774f553f1fe3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.877233193Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.881280372Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions","method":"POST","req_id":"49545880-cf6d-40fa-96ef-9a9fec3aab66","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21956,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.883684528Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.892256176Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.892291467Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"30104d9b-d8b7-4337-948d-774f553f1fe3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/30104d9b-d8b7-4337-948d-774f553f1fe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.955813743Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"30104d9b-d8b7-4337-948d-774f553f1fe3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"30104d9b-d8b7-4337-948d-774f553f1fe3\" \"--data\" \"/data/regions/30104d9b-d8b7-4337-948d-774f553f1fe3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:25.955868943Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"30104d9b-d8b7-4337-948d-774f553f1fe3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.113029301Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions","method":"POST","req_id":"158bdd0f-3567-4e48-86db-458410708182","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.21371244Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"30104d9b-d8b7-4337-948d-774f553f1fe3","component":"worker"}
{"msg":"region 30104d9b-d8b7-4337-948d-774f553f1fe3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.213759281Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.213876433Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.214309769Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.217428092Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.230916478Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.233585954Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.234144412Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.234213673Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.234258374Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.234298124Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.238516383Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.248417729Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.529191042Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions","method":"POST","req_id":"3d28b872-fa52-4247-b5d4-9a6f913d79e8","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.802978189Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","method":"GET","req_id":"da6ff673-46d3-486a-8b9f-8640609744fb","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":96,"response_code":"200"}
{"msg":"path is \"/data/regions/30104d9b-d8b7-4337-948d-774f553f1fe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.806360846Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"dataset is oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.817471299Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"region 30104d9b-d8b7-4337-948d-774f553f1fe3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.827505197Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.827656849Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.82771612Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","method":"DELETE","req_id":"69cfa4d5-af57-4c7b-afee-79b5ae6f5bb1","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21383,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-30104d9b-d8b7-4337-948d-774f553f1fe3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.828786525Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.831421112Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","method":"GET","req_id":"01c5f7f6-407d-4d89-a0b1-fcaa21776347","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.834356262Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible/regions/30104d9b-d8b7-4337-948d-774f553f1fe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.845745209Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"30104d9b-d8b7-4337-948d-774f553f1fe3","component":"worker"}
{"msg":"region 30104d9b-d8b7-4337-948d-774f553f1fe3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.919343854Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:26.920665993Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions/30104d9b-d8b7-4337-948d-774f553f1fe3","method":"GET","req_id":"337cb70d-e8af-46ce-8775-1aec4e7a7fe5","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"region 1e4c5e20-10ce-45b2-bbec-d11c3baf17e8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.15956041Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.159693052Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions","method":"POST","req_id":"fe2e1e26-5385-4288-880f-e28fe2bfc26b","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":197,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.159719212Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible/regions/1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.169455456Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b635d579-12bb-453a-8e41-991d0148fdbe/crucible/regions/1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.169486007Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1e4c5e20-10ce-45b2-bbec-d11c3baf17e8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1e4c5e20-10ce-45b2-bbec-d11c3baf17e8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.230847353Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1e4c5e20-10ce-45b2-bbec-d11c3baf17e8\" \"--data\" \"/data/regions/1e4c5e20-10ce-45b2-bbec-d11c3baf17e8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.232935882Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.306305084Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"region":"1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","component":"worker"}
{"msg":"region 1e4c5e20-10ce-45b2-bbec-d11c3baf17e8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.306345915Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.306438416Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.307797735Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.311219802Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.319406225Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.322525068Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1e4c5e20-10ce-45b2-bbec-d11c3baf17e8","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.322728421Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.322779911Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.322823222Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.322862123Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.326903888Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.332290143Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-18T18:36:29.481795544Z","hostname":"oxz_crucible_51a592ca-2d62-4777-9124-38b73956034f","pid":24030,"uri":"/crucible/0/regions","method":"POST","req_id":"f89bbbad-dc0d-434e-8ca1-e428bea73dd1","remote_addr":"[fd00:1122:3344:101::c]:37445","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":54,"response_code":"200"}