[ Apr 29 22:38:25 Enabled. ] [ Apr 29 22:38:25 Rereading configuration. ] [ Apr 29 22:38:27 Rereading configuration. ] [ Apr 29 22:38:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.678265204Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.680517035Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.680535795Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.697774388Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:31.848190555Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.907270217Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61012"} {"msg":"region 4caeaa36-99be-4ffb-9300-3196196260ae state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.967336916Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.968160045Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions","method":"POST","req_id":"3e2d2a06-8f96-4067-b589-e8d3e1eda502","remote_addr":"[fd00:1122:3344:101::c]:61012","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":1632,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.968185245Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible/regions/4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.977699612Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible/regions/4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:51.977745183Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"4caeaa36-99be-4ffb-9300-3196196260ae\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4caeaa36-99be-4ffb-9300-3196196260ae\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.030976226Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"4caeaa36-99be-4ffb-9300-3196196260ae","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4caeaa36-99be-4ffb-9300-3196196260ae\" \"--data\" \"/data/regions/4caeaa36-99be-4ffb-9300-3196196260ae\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.031046337Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"4caeaa36-99be-4ffb-9300-3196196260ae","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.144790378Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"4caeaa36-99be-4ffb-9300-3196196260ae","component":"worker"} {"msg":"region 4caeaa36-99be-4ffb-9300-3196196260ae state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.144839129Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.144936151Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.145383202Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.148750349Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.161075309Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.16419438Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164856305Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.164960538Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.165008969Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.16505053Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.168409246Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.173794419Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.212876689Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions","method":"POST","req_id":"d0a63457-9366-4cf2-afd5-3a0e1f344273","remote_addr":"[fd00:1122:3344:101::c]:61012","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.750063366Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:40739"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.751173362Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions/4caeaa36-99be-4ffb-9300-3196196260ae","method":"GET","req_id":"2eb67f81-6ade-4b90-8633-93b5e8154a66","remote_addr":"[fd00:1122:3344:101::c]:40739","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":1057,"response_code":"200"} {"msg":"path is \"/data/regions/4caeaa36-99be-4ffb-9300-3196196260ae\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.751385786Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"dataset is oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible/regions/4caeaa36-99be-4ffb-9300-3196196260ae","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.761113518Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"region 4caeaa36-99be-4ffb-9300-3196196260ae state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.769563971Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.769715504Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.769829737Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions/4caeaa36-99be-4ffb-9300-3196196260ae","method":"DELETE","req_id":"4f8bd5a4-a1c5-44ed-bda7-66ceebf8c5c1","remote_addr":"[fd00:1122:3344:101::c]:40739","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18493,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.770002661Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions/4caeaa36-99be-4ffb-9300-3196196260ae","method":"GET","req_id":"14f26099-67e9-405d-8de7-c7aefdde4411","remote_addr":"[fd00:1122:3344:101::c]:40739","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-4caeaa36-99be-4ffb-9300-3196196260ae (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.77083607Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.774399661Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible/regions/4caeaa36-99be-4ffb-9300-3196196260ae\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.783044338Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"4caeaa36-99be-4ffb-9300-3196196260ae","component":"worker"} {"msg":"region 4caeaa36-99be-4ffb-9300-3196196260ae state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.836223659Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:38:52.85339019Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions/4caeaa36-99be-4ffb-9300-3196196260ae","method":"GET","req_id":"52f6fdad-ecbc-4da9-b8a4-a360efff3d14","remote_addr":"[fd00:1122:3344:101::c]:40739","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.556572737Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42376"} {"msg":"region 260e4942-7b09-434d-a5f3-d2fac1525164 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.55667122Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.556870534Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions","method":"POST","req_id":"8e8518c6-f7bf-443f-9d5a-2bea9b4ca83b","remote_addr":"[fd00:1122:3344:101::a]:42376","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":239,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.556889725Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible/regions/260e4942-7b09-434d-a5f3-d2fac1525164","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.565654134Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3c2c8854-3269-4025-ae74-138b796a9368/crucible/regions/260e4942-7b09-434d-a5f3-d2fac1525164","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.565695435Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"260e4942-7b09-434d-a5f3-d2fac1525164\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/260e4942-7b09-434d-a5f3-d2fac1525164\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.617644999Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"260e4942-7b09-434d-a5f3-d2fac1525164","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"260e4942-7b09-434d-a5f3-d2fac1525164\" \"--data\" \"/data/regions/260e4942-7b09-434d-a5f3-d2fac1525164\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.619574013Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"260e4942-7b09-434d-a5f3-d2fac1525164","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.684724707Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"region":"260e4942-7b09-434d-a5f3-d2fac1525164","component":"worker"} {"msg":"region 260e4942-7b09-434d-a5f3-d2fac1525164 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.684777248Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.6848634Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-260e4942-7b09-434d-a5f3-d2fac1525164","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.686242811Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-260e4942-7b09-434d-a5f3-d2fac1525164","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.689183428Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.696459374Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-260e4942-7b09-434d-a5f3-d2fac1525164","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.69934268Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/260e4942-7b09-434d-a5f3-d2fac1525164","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.699723368Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.699793Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.699835691Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.699879522Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.703652128Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.708329394Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-29T22:39:43.764535805Z","hostname":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","pid":25386,"uri":"/crucible/0/regions","method":"POST","req_id":"e25fab7b-4140-44a0-b614-b8484b4a5d49","remote_addr":"[fd00:1122:3344:101::a]:42376","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":74,"response_code":"200"}