[ Apr 15 04:00:10 Enabled. ]
[ Apr 15 04:00:10 Rereading configuration. ]
[ Apr 15 04:00:12 Rereading configuration. ]
[ Apr 15 04:00:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f1a37602-e89e-4e31-bd14-3ce624f39e6c/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f1a37602-e89e-4e31-bd14-3ce624f39e6c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:15.722855003Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:15.726306627Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:15.726316297Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:15.746963837Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:15.863888332Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.653149463Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54815"}
{"msg":"region 737f91bb-b897-40e6-8763-c849bfc7787d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.696046068Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.699387769Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions","method":"POST","req_id":"26cd1c94-9ef1-4399-897d-c3a62df02805","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18124,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.701628617Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_f1a37602-e89e-4e31-bd14-3ce624f39e6c/crucible/regions/737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.712277064Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f1a37602-e89e-4e31-bd14-3ce624f39e6c/crucible/regions/737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.712302615Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"737f91bb-b897-40e6-8763-c849bfc7787d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/737f91bb-b897-40e6-8763-c849bfc7787d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.786457537Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"region":"737f91bb-b897-40e6-8763-c849bfc7787d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"737f91bb-b897-40e6-8763-c849bfc7787d\" \"--data\" \"/data/regions/737f91bb-b897-40e6-8763-c849bfc7787d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.788799647Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"region":"737f91bb-b897-40e6-8763-c849bfc7787d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.99184559Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions","method":"POST","req_id":"944f5f10-83e2-480c-b083-7607c44d156f","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.154672184Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"region":"737f91bb-b897-40e6-8763-c849bfc7787d","component":"worker"}
{"msg":"region 737f91bb-b897-40e6-8763-c849bfc7787d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.154723185Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.155408889Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.155815988Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.16995633Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.201203407Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.231506433Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.234371565Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.234463447Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.234509397Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.234548728Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.241443375Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.251133692Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.403001623Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions","method":"POST","req_id":"f6bbdad5-f3a3-4d1f-890f-54db5fd55f5d","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.548329219Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions/737f91bb-b897-40e6-8763-c849bfc7787d","method":"GET","req_id":"f416f23d-c681-4c08-a3d4-611f133ccf42","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2422,"response_code":"200"}
{"msg":"path is \"/data/regions/737f91bb-b897-40e6-8763-c849bfc7787d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.551045597Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"dataset is oxp_f1a37602-e89e-4e31-bd14-3ce624f39e6c/crucible/regions/737f91bb-b897-40e6-8763-c849bfc7787d","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.564386632Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"region 737f91bb-b897-40e6-8763-c849bfc7787d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.574047988Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.574372205Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.574432366Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions/737f91bb-b897-40e6-8763-c849bfc7787d","method":"DELETE","req_id":"883334ea-d5de-4861-b85f-b58f1d107cf3","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":23409,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-737f91bb-b897-40e6-8763-c849bfc7787d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.575451328Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.577714926Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions/737f91bb-b897-40e6-8763-c849bfc7787d","method":"GET","req_id":"004c9388-95e3-41a8-a0d8-e6b453c04f6c","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.585781498Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f1a37602-e89e-4e31-bd14-3ce624f39e6c/crucible/regions/737f91bb-b897-40e6-8763-c849bfc7787d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.597809355Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"region":"737f91bb-b897-40e6-8763-c849bfc7787d","component":"worker"}
{"msg":"region 737f91bb-b897-40e6-8763-c849bfc7787d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.653005753Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.725131132Z","hostname":"oxz_crucible_e21ff481-7b83-4509-b070-53dfc310b802","pid":25578,"uri":"/crucible/0/regions/737f91bb-b897-40e6-8763-c849bfc7787d","method":"GET","req_id":"c43af0dc-2698-43cb-afb9-05eff1a89274","remote_addr":"[fd00:1122:3344:101::c]:54815","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":"200"}