[ Apr 15 04:00:09 Enabled. ]
[ Apr 15 04:00:09 Rereading configuration. ]
[ Apr 15 04:00:10 Rereading configuration. ]
[ Apr 15 04:00:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1bf6bee1-3707-4f13-a189-5c6e8cd0b4ce/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1bf6bee1-3707-4f13-a189-5c6e8cd0b4ce/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:13.74497592Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:13.745303387Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:13.745312367Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:13.760359008Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:13.88933108Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:11.96874074Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60642"}
{"msg":"region bf13c741-3489-466f-bcbb-f5647d1dba9f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.003866659Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.00903381Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions","method":"POST","req_id":"cb038c3d-376c-48e2-9f92-72734d3d22c4","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19140,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.011560343Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_1bf6bee1-3707-4f13-a189-5c6e8cd0b4ce/crucible/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.021758141Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_1bf6bee1-3707-4f13-a189-5c6e8cd0b4ce/crucible/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.021803272Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bf13c741-3489-466f-bcbb-f5647d1dba9f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.076957049Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"region":"bf13c741-3489-466f-bcbb-f5647d1dba9f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bf13c741-3489-466f-bcbb-f5647d1dba9f\" \"--data\" \"/data/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.079265118Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"region":"bf13c741-3489-466f-bcbb-f5647d1dba9f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.222252729Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions","method":"POST","req_id":"41f502fd-c20a-42b2-a0ab-044f92d05d90","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.394930954Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"region":"bf13c741-3489-466f-bcbb-f5647d1dba9f","component":"worker"}
{"msg":"region bf13c741-3489-466f-bcbb-f5647d1dba9f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.394978295Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.395056736Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.395480585Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.400801829Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.418821614Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.434918147Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.435482939Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.435580271Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.435645853Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.435689263Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.440511836Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.446643267Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:12.858315621Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions","method":"POST","req_id":"ce0bbfd9-a0dd-4658-80a4-7c1708a817b7","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.253109385Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","method":"GET","req_id":"d687a334-1a27-4d08-bebd-8ca084562528","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"path is \"/data/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.256463697Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"dataset is oxp_1bf6bee1-3707-4f13-a189-5c6e8cd0b4ce/crucible/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.267530943Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"region bf13c741-3489-466f-bcbb-f5647d1dba9f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.277625508Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.277782502Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.277816243Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","method":"DELETE","req_id":"d734d92c-b95f-4574-89ed-d2c75a0fc760","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21390,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-bf13c741-3489-466f-bcbb-f5647d1dba9f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.278966007Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.281295647Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","method":"GET","req_id":"8e509f95-efe8-43d2-aaf1-f871ceaea94f","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.283645017Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1bf6bee1-3707-4f13-a189-5c6e8cd0b4ce/crucible/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.29505775Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"region":"bf13c741-3489-466f-bcbb-f5647d1dba9f","component":"worker"}
{"msg":"region bf13c741-3489-466f-bcbb-f5647d1dba9f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.363401089Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:01:13.418733709Z","hostname":"oxz_crucible_627fc6e4-9c1a-48d2-8748-8c5f1653f778","pid":24703,"uri":"/crucible/0/regions/bf13c741-3489-466f-bcbb-f5647d1dba9f","method":"GET","req_id":"61844477-0268-4750-b967-272d46b48007","remote_addr":"[fd00:1122:3344:101::c]:60642","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":61,"response_code":"200"}