[ Apr 15 04:00:12 Enabled. ]
[ Apr 15 04:00:12 Rereading configuration. ]
[ Apr 15 04:00:14 Rereading configuration. ]
[ Apr 15 04:00:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d59d9b6a-98de-4563-9202-2118f03d0e71/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d59d9b6a-98de-4563-9202-2118f03d0e71/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:18.038900981Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:18.0421429Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:18.04215222Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:18.058200603Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:00:18.137460064Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.652828436Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:35214"}
{"msg":"region 130a7c57-a3de-46ac-8ae3-927c4ee37cd3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.698843138Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.704251783Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions","method":"POST","req_id":"8068247b-52db-49ba-ab5f-f0f7140bb04a","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22817,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.706257746Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d59d9b6a-98de-4563-9202-2118f03d0e71/crucible/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.718276632Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d59d9b6a-98de-4563-9202-2118f03d0e71/crucible/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.718311243Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"130a7c57-a3de-46ac-8ae3-927c4ee37cd3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.778817194Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"region":"130a7c57-a3de-46ac-8ae3-927c4ee37cd3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"130a7c57-a3de-46ac-8ae3-927c4ee37cd3\" \"--data\" \"/data/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:48.781140124Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"region":"130a7c57-a3de-46ac-8ae3-927c4ee37cd3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.030888723Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions","method":"POST","req_id":"184a0969-363b-4728-918a-f2c93e59db51","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.161415218Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"region":"130a7c57-a3de-46ac-8ae3-927c4ee37cd3","component":"worker"}
{"msg":"region 130a7c57-a3de-46ac-8ae3-927c4ee37cd3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.161462019Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.161583511Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.162058442Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.175834336Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.201121275Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.227193341Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.22995494Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.230041182Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.230086803Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.230126504Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.234702801Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.243667783Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:11:49.658454014Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions","method":"POST","req_id":"63212735-3d4f-4d36-88d1-d48caf49b237","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.547546332Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","method":"GET","req_id":"de668e31-47f2-4a52-bf81-aadf2df53054","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":3001,"response_code":"200"}
{"msg":"path is \"/data/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.550386273Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"dataset is oxp_d59d9b6a-98de-4563-9202-2118f03d0e71/crucible/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.562626144Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"region 130a7c57-a3de-46ac-8ae3-927c4ee37cd3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.572485515Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.57321779Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.573261351Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","method":"DELETE","req_id":"92fbdc19-e8f7-476f-991a-3d090b4705e5","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22914,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-130a7c57-a3de-46ac-8ae3-927c4ee37cd3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.574477067Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.577488831Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","method":"GET","req_id":"cf9dbc27-e3fb-4a40-8d9f-da8c4c45c6be","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.586618056Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d59d9b6a-98de-4563-9202-2118f03d0e71/crucible/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.598156582Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"region":"130a7c57-a3de-46ac-8ae3-927c4ee37cd3","component":"worker"}
{"msg":"region 130a7c57-a3de-46ac-8ae3-927c4ee37cd3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.6533578Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-15T04:12:14.661510104Z","hostname":"oxz_crucible_bda5b8d7-4cb6-4d6f-aac3-de07e95da9fb","pid":26965,"uri":"/crucible/0/regions/130a7c57-a3de-46ac-8ae3-927c4ee37cd3","method":"GET","req_id":"de8448b9-9e3e-4e22-b0dc-0ef63f8f55b1","remote_addr":"[fd00:1122:3344:101::c]:35214","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":63,"response_code":"200"}