[ Mar 26 00:01:32 Enabled. ] [ Mar 26 00:01:32 Rereading configuration. ] [ Mar 26 00:01:34 Rereading configuration. ] [ Mar 26 00:01:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.264921681Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490} {"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.265667065Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.265675806Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.283775037Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.37549291Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.675988336Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:57852"} {"msg":"region 1bf48e39-bd39-4dd1-9163-8267694828fc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.687502776Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.688018232Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions","method":"POST","req_id":"c4efe77e-0a32-45bb-a0f3-d04c32b001e3","remote_addr":"[fd00:1122:3344:101::b]:57852","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":5515,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.688594431Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.697935201Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.697995783Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"1bf48e39-bd39-4dd1-9163-8267694828fc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1bf48e39-bd39-4dd1-9163-8267694828fc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.771483681Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"1bf48e39-bd39-4dd1-9163-8267694828fc","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1bf48e39-bd39-4dd1-9163-8267694828fc\" \"--data\" \"/data/regions/1bf48e39-bd39-4dd1-9163-8267694828fc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.773468185Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"1bf48e39-bd39-4dd1-9163-8267694828fc","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.911718483Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"1bf48e39-bd39-4dd1-9163-8267694828fc","component":"worker"} {"msg":"region 1bf48e39-bd39-4dd1-9163-8267694828fc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.911814436Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.912049903Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.912482987Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.916463705Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.925643139Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.929604757Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.930376262Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.930438884Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.930481585Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.930524906Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.930743493Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions","method":"POST","req_id":"3cd78fd9-f332-4187-99cd-7915633bc8ab","remote_addr":"[fd00:1122:3344:101::b]:57852","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.936441086Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:51.943596156Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.374943631Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","method":"GET","req_id":"3a258ad2-10ba-4bee-ad7c-ecd3c002f974","remote_addr":"[fd00:1122:3344:101::b]:57852","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":64,"response_code":"200"} {"msg":"path is \"/data/regions/1bf48e39-bd39-4dd1-9163-8267694828fc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.377877226Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"dataset is oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.389671594Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"region 1bf48e39-bd39-4dd1-9163-8267694828fc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.399551611Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.399763688Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.39981427Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","method":"DELETE","req_id":"520593db-3fe4-4a65-add8-cbadd2e8a7d3","remote_addr":"[fd00:1122:3344:101::b]:57852","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21962,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-1bf48e39-bd39-4dd1-9163-8267694828fc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.400965217Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.403314102Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","method":"GET","req_id":"e1776c15-cbd8-44aa-b772-e5cc5a94cac7","remote_addr":"[fd00:1122:3344:101::b]:57852","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":29,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.406880786Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible/regions/1bf48e39-bd39-4dd1-9163-8267694828fc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.419253163Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"1bf48e39-bd39-4dd1-9163-8267694828fc","component":"worker"} {"msg":"region 1bf48e39-bd39-4dd1-9163-8267694828fc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.491994549Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:52.522783347Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions/1bf48e39-bd39-4dd1-9163-8267694828fc","method":"GET","req_id":"bd3c860d-bfcd-478f-be1d-3b8a97f29362","remote_addr":"[fd00:1122:3344:101::b]:57852","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.737252714Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38738"} {"msg":"region 6533fc40-6a97-486d-9ed9-4a50222dd417 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.737405529Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.738157493Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.738173763Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions","method":"POST","req_id":"7f78590f-c610-4f95-a137-bf659d47c26e","remote_addr":"[fd00:1122:3344:101::b]:38738","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":848,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible/regions/6533fc40-6a97-486d-9ed9-4a50222dd417","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.749801427Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_73016347-d5cb-4e45-a92b-78d919a742ab/crucible/regions/6533fc40-6a97-486d-9ed9-4a50222dd417","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.749834688Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6533fc40-6a97-486d-9ed9-4a50222dd417\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6533fc40-6a97-486d-9ed9-4a50222dd417\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.810860527Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"6533fc40-6a97-486d-9ed9-4a50222dd417","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6533fc40-6a97-486d-9ed9-4a50222dd417\" \"--data\" \"/data/regions/6533fc40-6a97-486d-9ed9-4a50222dd417\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.812862681Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"6533fc40-6a97-486d-9ed9-4a50222dd417","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.902593221Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"region":"6533fc40-6a97-486d-9ed9-4a50222dd417","component":"worker"} {"msg":"region 6533fc40-6a97-486d-9ed9-4a50222dd417 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.902622092Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.902722355Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-6533fc40-6a97-486d-9ed9-4a50222dd417","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.904064338Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6533fc40-6a97-486d-9ed9-4a50222dd417","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.912902492Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.926030693Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"uri":"/crucible/0/regions","method":"POST","req_id":"060d417c-2355-428e-8d9c-375560e1e528","remote_addr":"[fd00:1122:3344:101::b]:38738","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.927717787Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6533fc40-6a97-486d-9ed9-4a50222dd417","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.933050739Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6533fc40-6a97-486d-9ed9-4a50222dd417","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.933361919Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.933425511Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.933470002Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.933513693Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.937563314Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:56.942474401Z","hostname":"oxz_crucible_64182428-2a14-4d58-b6fc-ca4fc58ae5ff","pid":26490,"component":"worker"}