[ Oct 15 20:57:59 Enabled. ]
[ Oct 15 20:57:59 Rereading configuration. ]
[ Oct 15 20:58:04 Rereading configuration. ]
[ Oct 15 20:58:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_66ab2565-ef03-4534-89ac-937d627e1e68/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 65ca41e821ef53ec9c28909357f23e3348169e4f\nCommit timestamp: 2025-09-17T20:55:30.000000000Z  branch: main\nrustc: 1.86.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:58:09.626406107Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321}
{"msg":"dataset: \"oxp_66ab2565-ef03-4534-89ac-937d627e1e68/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:58:09.627294845Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:58:09.627351637Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:58:09.627363238Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:58:09.647256675Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:58:09.76258223Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.312151674Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:57879"}
{"msg":"region b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.347831557Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.355621027Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions","method":"POST","req_id":"483e5e8e-ed28-404b-9f94-6b126a212f6f","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24952,"response_code":200}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.358050885Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_66ab2565-ef03-4534-89ac-937d627e1e68/crucible/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.368251611Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_66ab2565-ef03-4534-89ac-937d627e1e68/crucible/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.368287042Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.461501719Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"region":"b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0\" \"--data\" \"/data/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.463879965Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"region":"b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.737096989Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions","method":"POST","req_id":"9b50ac6c-36fd-4cbe-b423-db385ceed49c","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.780116757Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"region":"b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","component":"worker"}
{"msg":"region b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.780150058Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.780265942Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.780695126Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.786313336Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.803776566Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.807148343Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.80798072Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.808040502Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.808080833Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.808117005Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.811849374Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:03.816004307Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.376585668Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions","method":"POST","req_id":"9e7438b8-d8df-4113-9328-3ff57dd54eb0","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":48,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.753725661Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","method":"GET","req_id":"2e014ab4-5e0e-49c4-9343-4dcec1b283d2","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1971,"response_code":200}
{"msg":"path is \"/data/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.757069358Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"dataset is oxp_66ab2565-ef03-4534-89ac-937d627e1e68/crucible/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.768960479Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"region b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.781249193Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.781409428Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.78146255Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","method":"DELETE","req_id":"5ca1c565-7306-44cb-b4fe-35739c1995e0","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24436,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.782440065Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.785807689Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","method":"GET","req_id":"a8ed37ca-545b-418f-9829-88cc899b4032","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":34,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.786696398Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_66ab2565-ef03-4534-89ac-937d627e1e68/crucible/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.79894931Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"region":"b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","component":"worker"}
{"msg":"region b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.864578383Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-15T20:59:04.90504761Z","hostname":"oxz_crucible_8639697c-99aa-4182-9c01-463b50540ef4","pid":23321,"uri":"/crucible/0/regions/b7fc23d2-bae4-4f42-bf1a-b1c555e4f3e0","method":"GET","req_id":"edcddc86-003e-4fd8-bb6f-fe353079ba22","remote_addr":"[fd00:1122:3344:101::b]:57879","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":62,"response_code":200}