[ Feb 29 22:26:02 Enabled. ]
[ Feb 29 22:26:02 Rereading configuration. ]
[ Feb 29 22:26:05 Rereading configuration. ]
[ Feb 29 22:26:06 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:06.946163471Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:06.947093155Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:06.947103315Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:07.092952911Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.809431567Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:52383"}
{"msg":"region ed7b38ba-aab8-4572-a394-db0ba38dcf62 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.83758691Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.840332781Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions","method":"POST","req_id":"69b84b84-5518-4f94-a494-d474bc9c5d89","remote_addr":"[fd00:1122:3344:101::b]:52383","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":15228,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.840969667Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.848934592Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.849020814Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ed7b38ba-aab8-4572-a394-db0ba38dcf62\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:47.986107444Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"region":"ed7b38ba-aab8-4572-a394-db0ba38dcf62","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.091222764Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions","method":"POST","req_id":"00c3cd7b-5924-4c79-a847-57c39116199a","remote_addr":"[fd00:1122:3344:101::b]:52383","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":746,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.292565895Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"region":"ed7b38ba-aab8-4572-a394-db0ba38dcf62","component":"worker"}
{"msg":"region ed7b38ba-aab8-4572-a394-db0ba38dcf62 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.292603216Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.292898823Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.293318984Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.319024185Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.36363958Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.382088704Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.414334252Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.416496558Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.416560489Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.41660199Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.433965106Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.455022517Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:48.799487034Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions","method":"POST","req_id":"04c92002-8c1f-4f3c-9bf6-de7773f8f17f","remote_addr":"[fd00:1122:3344:101::b]:52383","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.386843679Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61422"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.388896982Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","method":"GET","req_id":"cd44a94c-4a5c-47af-8cf2-52579e9a610b","remote_addr":"[fd00:1122:3344:101::b]:61422","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2016,"response_code":"200"}
{"msg":"path is \"/data/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.389093457Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.398792026Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"region ed7b38ba-aab8-4572-a394-db0ba38dcf62 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.407973182Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.408207798Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.40827888Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","method":"DELETE","req_id":"d0274a54-bf35-482a-a738-375870818ca6","remote_addr":"[fd00:1122:3344:101::b]:61422","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19208,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.408434744Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","method":"GET","req_id":"34e5a749-b26e-4588-8dd2-5d3466f91984","remote_addr":"[fd00:1122:3344:101::b]:61422","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-ed7b38ba-aab8-4572-a394-db0ba38dcf62 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.409238494Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.444400767Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.455154064Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"region":"ed7b38ba-aab8-4572-a394-db0ba38dcf62","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.539766797Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","method":"GET","req_id":"cb96b43b-47bd-4595-abfc-ca48d3807d2f","remote_addr":"[fd00:1122:3344:101::b]:61422","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"region ed7b38ba-aab8-4572-a394-db0ba38dcf62 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.584418633Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:26:50.681055995Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions/ed7b38ba-aab8-4572-a394-db0ba38dcf62","method":"GET","req_id":"eaed57aa-48fa-4bf0-95d1-82dd0b132833","remote_addr":"[fd00:1122:3344:101::b]:61422","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.577504636Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58676"}
{"msg":"region 791562f7-d4c3-48d2-ae65-0e8d83f43dd6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.606639144Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.60686613Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions","method":"POST","req_id":"9a6bfda3-cd87-4b41-894e-721f4b082f15","remote_addr":"[fd00:1122:3344:101::a]:58676","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":274,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.606888921Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/791562f7-d4c3-48d2-ae65-0e8d83f43dd6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.614867296Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/791562f7-d4c3-48d2-ae65-0e8d83f43dd6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.614898116Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"791562f7-d4c3-48d2-ae65-0e8d83f43dd6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/791562f7-d4c3-48d2-ae65-0e8d83f43dd6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.692391347Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"region":"791562f7-d4c3-48d2-ae65-0e8d83f43dd6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.811710671Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"region":"791562f7-d4c3-48d2-ae65-0e8d83f43dd6","component":"worker"}
{"msg":"region 791562f7-d4c3-48d2-ae65-0e8d83f43dd6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.811761852Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.811861735Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-791562f7-d4c3-48d2-ae65-0e8d83f43dd6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.813361193Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-791562f7-d4c3-48d2-ae65-0e8d83f43dd6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.820468506Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.838854488Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-791562f7-d4c3-48d2-ae65-0e8d83f43dd6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.848622778Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/791562f7-d4c3-48d2-ae65-0e8d83f43dd6","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.848912466Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.848986278Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.849039869Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.84909155Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.854985182Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"uri":"/crucible/0/regions","method":"POST","req_id":"7d06b848-b169-4ad8-9342-8fd365e653bb","remote_addr":"[fd00:1122:3344:101::a]:58676","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.856207253Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-29T22:27:29.864566278Z","hostname":"oxz_crucible_18bb87bd-7740-4917-b08a-e0e128c3e68d","pid":21019,"component":"worker"}