2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Disclaimer: The codes contained in these modules may be specific to
19 * the Intel Software Development Platform codenamed: Knights Ferry, and
20 * the Intel product codenamed: Knights Corner, and are not backward
21 * compatible with other Intel products. Additionally, Intel will NOT
22 * support the codes or instruction set in future products.
26 * virtio for kvm on s390
28 * Copyright IBM Corp. 2008
30 * This program is free software; you can redistribute it and/or modify
31 * it under the terms of the GNU General Public License (version 2 only)
32 * as published by the Free Software Foundation.
34 * Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
36 * Intel MIC Card driver.
39 #include <linux/delay.h>
40 #include <linux/slab.h>
41 #include <linux/virtio_config.h>
43 #include "../common/mic_dev.h"
44 #include "mic_virtio.h"
46 #define VIRTIO_SUBCODE_64 0x0D00
48 #define MIC_MAX_VRINGS 4
50 struct virtio_device vdev;
51 struct mic_device_desc __iomem *desc;
52 struct mic_device_ctrl __iomem *dc;
53 struct mic_device *mdev;
54 void __iomem *vr[MIC_MAX_VRINGS];
55 int used_size[MIC_MAX_VRINGS];
56 struct completion reset_done;
57 struct mic_irq *virtio_cookie;
61 static struct mic_irq *virtio_config_cookie;
62 #define to_micvdev(vd) container_of(vd, struct mic_vdev, vdev)
64 /* Helper API to obtain the parent of the virtio device */
65 static inline struct device *mic_dev(struct mic_vdev *mvdev)
67 return mvdev->vdev.dev.parent;
70 /* This gets the device's feature bits. */
71 static u64 mic_get_features(struct virtio_device *vdev)
75 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
76 u8 __iomem *in_features = mic_vq_features(desc);
77 int feature_len = ioread8(&desc->feature_len);
79 bits = min_t(unsigned, feature_len, sizeof(features)) * 8;
80 for (i = 0; i < bits; i++)
81 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
87 static void mic_finalize_features(struct virtio_device *vdev)
90 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
91 u8 feature_len = ioread8(&desc->feature_len);
92 /* Second half of bitmap is features we accept. */
93 u8 __iomem *out_features =
94 mic_vq_features(desc) + feature_len;
96 /* Give virtio_ring a chance to accept features. */
97 vring_transport_features(vdev);
99 /* Make sure we don't have any features > 32 bits! */
100 BUG_ON((u32)vdev->features != vdev->features);
102 memset_io(out_features, 0, feature_len);
103 bits = min_t(unsigned, feature_len,
104 sizeof(vdev->features)) * 8;
105 for (i = 0; i < bits; i++) {
106 if (__virtio_test_bit(vdev, i))
107 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
108 &out_features[i / 8]);
113 * Reading and writing elements in config space
115 static void mic_get(struct virtio_device *vdev, unsigned int offset,
116 void *buf, unsigned len)
118 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
120 if (offset + len > ioread8(&desc->config_len))
122 memcpy_fromio(buf, mic_vq_configspace(desc) + offset, len);
125 static void mic_set(struct virtio_device *vdev, unsigned int offset,
126 const void *buf, unsigned len)
128 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
130 if (offset + len > ioread8(&desc->config_len))
132 memcpy_toio(mic_vq_configspace(desc) + offset, buf, len);
136 * The operations to get and set the status word just access the status
137 * field of the device descriptor. set_status also interrupts the host
138 * to tell about status changes.
140 static u8 mic_get_status(struct virtio_device *vdev)
142 return ioread8(&to_micvdev(vdev)->desc->status);
145 static void mic_set_status(struct virtio_device *vdev, u8 status)
147 struct mic_vdev *mvdev = to_micvdev(vdev);
150 iowrite8(status, &mvdev->desc->status);
151 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
154 /* Inform host on a virtio device reset and wait for ack from host */
155 static void mic_reset_inform_host(struct virtio_device *vdev)
157 struct mic_vdev *mvdev = to_micvdev(vdev);
158 struct mic_device_ctrl __iomem *dc = mvdev->dc;
161 iowrite8(0, &dc->host_ack);
162 iowrite8(1, &dc->vdev_reset);
163 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
165 /* Wait till host completes all card accesses and acks the reset */
166 for (retry = 100; retry--;) {
167 if (ioread8(&dc->host_ack))
172 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry);
174 /* Reset status to 0 in case we timed out */
175 iowrite8(0, &mvdev->desc->status);
178 static void mic_reset(struct virtio_device *vdev)
180 struct mic_vdev *mvdev = to_micvdev(vdev);
182 dev_dbg(mic_dev(mvdev), "%s: virtio id %d\n",
183 __func__, vdev->id.device);
185 mic_reset_inform_host(vdev);
186 complete_all(&mvdev->reset_done);
190 * The virtio_ring code calls this API when it wants to notify the Host.
192 static bool mic_notify(struct virtqueue *vq)
194 struct mic_vdev *mvdev = vq->priv;
196 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
200 static void mic_del_vq(struct virtqueue *vq, int n)
202 struct mic_vdev *mvdev = to_micvdev(vq->vdev);
203 struct vring *vr = (struct vring *)(vq + 1);
205 free_pages((unsigned long) vr->used, get_order(mvdev->used_size[n]));
206 vring_del_virtqueue(vq);
207 mic_card_unmap(mvdev->mdev, mvdev->vr[n]);
211 static void mic_del_vqs(struct virtio_device *vdev)
213 struct mic_vdev *mvdev = to_micvdev(vdev);
214 struct virtqueue *vq, *n;
217 dev_dbg(mic_dev(mvdev), "%s\n", __func__);
219 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
220 mic_del_vq(vq, idx++);
224 * This routine will assign vring's allocated in host/io memory. Code in
225 * virtio_ring.c however continues to access this io memory as if it were local
226 * memory without io accessors.
228 static struct virtqueue *mic_find_vq(struct virtio_device *vdev,
230 void (*callback)(struct virtqueue *vq),
233 struct mic_vdev *mvdev = to_micvdev(vdev);
234 struct mic_vqconfig __iomem *vqconfig;
235 struct mic_vqconfig config;
236 struct virtqueue *vq;
238 struct _mic_vring_info __iomem *info;
240 int vr_size, _vr_size, err, magic;
242 u8 type = ioread8(&mvdev->desc->type);
244 if (index >= ioread8(&mvdev->desc->num_vq))
245 return ERR_PTR(-ENOENT);
248 return ERR_PTR(-ENOENT);
250 /* First assign the vring's allocated in host memory */
251 vqconfig = mic_vq_config(mvdev->desc) + index;
252 memcpy_fromio(&config, vqconfig, sizeof(config));
253 _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
254 vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
255 va = mic_card_map(mvdev->mdev, le64_to_cpu(config.address), vr_size);
257 return ERR_PTR(-ENOMEM);
258 mvdev->vr[index] = va;
259 memset_io(va, 0x0, _vr_size);
260 vq = vring_new_virtqueue(index, le16_to_cpu(config.num),
261 MIC_VIRTIO_RING_ALIGN, vdev, false,
262 (void __force *)va, mic_notify, callback,
268 info = va + _vr_size;
269 magic = ioread32(&info->magic);
271 if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
276 /* Allocate and reassign used ring now */
277 mvdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
278 sizeof(struct vring_used_elem) *
279 le16_to_cpu(config.num));
280 used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
281 get_order(mvdev->used_size[index]));
284 dev_err(mic_dev(mvdev), "%s %d err %d\n",
285 __func__, __LINE__, err);
288 iowrite64(virt_to_phys(used), &vqconfig->used_address);
291 * To reassign the used ring here we are directly accessing
292 * struct vring_virtqueue which is a private data structure
293 * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in
294 * vring_new_virtqueue() would ensure that
295 * (&vq->vring == (struct vring *) (&vq->vq + 1));
297 vr = (struct vring *)(vq + 1);
303 vring_del_virtqueue(vq);
305 mic_card_unmap(mvdev->mdev, mvdev->vr[index]);
309 static int mic_find_vqs(struct virtio_device *vdev, unsigned nvqs,
310 struct virtqueue *vqs[],
311 vq_callback_t *callbacks[],
314 struct mic_vdev *mvdev = to_micvdev(vdev);
315 struct mic_device_ctrl __iomem *dc = mvdev->dc;
318 /* We must have this many virtqueues. */
319 if (nvqs > ioread8(&mvdev->desc->num_vq))
322 for (i = 0; i < nvqs; ++i) {
323 dev_dbg(mic_dev(mvdev), "%s: %d: %s\n",
324 __func__, i, names[i]);
325 vqs[i] = mic_find_vq(vdev, i, callbacks[i], names[i]);
326 if (IS_ERR(vqs[i])) {
327 err = PTR_ERR(vqs[i]);
332 iowrite8(1, &dc->used_address_updated);
334 * Send an interrupt to the host to inform it that used
335 * rings have been re-assigned.
337 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
338 for (retry = 100; retry--;) {
339 if (!ioread8(&dc->used_address_updated))
344 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry);
357 * The config ops structure as defined by virtio config
359 static struct virtio_config_ops mic_vq_config_ops = {
360 .get_features = mic_get_features,
361 .finalize_features = mic_finalize_features,
364 .get_status = mic_get_status,
365 .set_status = mic_set_status,
367 .find_vqs = mic_find_vqs,
368 .del_vqs = mic_del_vqs,
372 mic_virtio_intr_handler(int irq, void *data)
374 struct mic_vdev *mvdev = data;
375 struct virtqueue *vq;
377 mic_ack_interrupt(mvdev->mdev);
378 list_for_each_entry(vq, &mvdev->vdev.vqs, list)
379 vring_interrupt(0, vq);
384 static void mic_virtio_release_dev(struct device *_d)
387 * No need for a release method similar to virtio PCI.
388 * Provide an empty one to avoid getting a warning from core.
393 * adds a new device and register it with virtio
394 * appropriate drivers are loaded by the device model
396 static int mic_add_device(struct mic_device_desc __iomem *d,
397 unsigned int offset, struct mic_driver *mdrv)
399 struct mic_vdev *mvdev;
402 u8 type = ioread8(&d->type);
404 mvdev = kzalloc(sizeof(*mvdev), GFP_KERNEL);
406 dev_err(mdrv->dev, "Cannot allocate mic dev %u type %u\n",
411 mvdev->mdev = &mdrv->mdev;
412 mvdev->vdev.dev.parent = mdrv->dev;
413 mvdev->vdev.dev.release = mic_virtio_release_dev;
414 mvdev->vdev.id.device = type;
415 mvdev->vdev.config = &mic_vq_config_ops;
417 mvdev->dc = (void __iomem *)d + mic_aligned_desc_size(d);
418 init_completion(&mvdev->reset_done);
420 virtio_db = mic_next_card_db();
421 mvdev->virtio_cookie = mic_request_card_irq(mic_virtio_intr_handler,
422 NULL, "virtio intr", mvdev, virtio_db);
423 if (IS_ERR(mvdev->virtio_cookie)) {
424 ret = PTR_ERR(mvdev->virtio_cookie);
427 iowrite8((u8)virtio_db, &mvdev->dc->h2c_vdev_db);
428 mvdev->c2h_vdev_db = ioread8(&mvdev->dc->c2h_vdev_db);
430 ret = register_virtio_device(&mvdev->vdev);
432 dev_err(mic_dev(mvdev),
433 "Failed to register mic device %u type %u\n",
437 iowrite64((u64)mvdev, &mvdev->dc->vdev);
438 dev_dbg(mic_dev(mvdev), "%s: registered mic device %u type %u mvdev %p\n",
439 __func__, offset, type, mvdev);
444 mic_free_card_irq(mvdev->virtio_cookie, mvdev);
451 * match for a mic device with a specific desc pointer
453 static int mic_match_desc(struct device *dev, void *data)
455 struct virtio_device *vdev = dev_to_virtio(dev);
456 struct mic_vdev *mvdev = to_micvdev(vdev);
458 return mvdev->desc == (void __iomem *)data;
461 static void mic_handle_config_change(struct mic_device_desc __iomem *d,
462 unsigned int offset, struct mic_driver *mdrv)
464 struct mic_device_ctrl __iomem *dc
465 = (void __iomem *)d + mic_aligned_desc_size(d);
466 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev);
468 if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
471 dev_dbg(mdrv->dev, "%s %d\n", __func__, __LINE__);
472 virtio_config_changed(&mvdev->vdev);
473 iowrite8(1, &dc->guest_ack);
477 * removes a virtio device if a hot remove event has been
478 * requested by the host.
480 static int mic_remove_device(struct mic_device_desc __iomem *d,
481 unsigned int offset, struct mic_driver *mdrv)
483 struct mic_device_ctrl __iomem *dc
484 = (void __iomem *)d + mic_aligned_desc_size(d);
485 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev);
489 if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) {
491 "%s %d config_change %d type %d mvdev %p\n",
493 ioread8(&dc->config_change), ioread8(&d->type), mvdev);
495 status = ioread8(&d->status);
496 reinit_completion(&mvdev->reset_done);
497 unregister_virtio_device(&mvdev->vdev);
498 mic_free_card_irq(mvdev->virtio_cookie, mvdev);
499 if (status & VIRTIO_CONFIG_S_DRIVER_OK)
500 wait_for_completion(&mvdev->reset_done);
502 iowrite8(1, &dc->guest_ack);
503 dev_dbg(mdrv->dev, "%s %d guest_ack %d\n",
504 __func__, __LINE__, ioread8(&dc->guest_ack));
511 #define REMOVE_DEVICES true
513 static void mic_scan_devices(struct mic_driver *mdrv, bool remove)
517 struct mic_device_desc __iomem *d;
518 struct mic_device_ctrl __iomem *dc;
522 for (i = sizeof(struct mic_bootparam); i < MIC_DP_SIZE;
523 i += mic_total_desc_size(d)) {
525 dc = (void __iomem *)d + mic_aligned_desc_size(d);
527 * This read barrier is paired with the corresponding write
528 * barrier on the host which is inserted before adding or
529 * removing a virtio device descriptor, by updating the type.
532 type = ioread8(&d->type);
541 /* device already exists */
542 dev = device_find_child(mdrv->dev, (void __force *)d,
546 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE,
549 mic_handle_config_change(d, i, mdrv);
550 ret = mic_remove_device(d, i, mdrv);
552 iowrite8(-1, &d->type);
554 iowrite8(0, &dc->config_change);
555 iowrite8(0, &dc->guest_ack);
561 dev_dbg(mdrv->dev, "%s %d Adding new virtio device %p\n",
562 __func__, __LINE__, d);
564 mic_add_device(d, i, mdrv);
569 * mic_hotplug_device tries to find changes in the device page.
571 static void mic_hotplug_devices(struct work_struct *work)
573 struct mic_driver *mdrv = container_of(work,
574 struct mic_driver, hotplug_work);
576 mic_scan_devices(mdrv, !REMOVE_DEVICES);
580 * Interrupt handler for hot plug/config changes etc.
583 mic_extint_handler(int irq, void *data)
585 struct mic_driver *mdrv = (struct mic_driver *)data;
587 dev_dbg(mdrv->dev, "%s %d hotplug work\n",
589 mic_ack_interrupt(&mdrv->mdev);
590 schedule_work(&mdrv->hotplug_work);
595 * Init function for virtio
597 int mic_devices_init(struct mic_driver *mdrv)
600 struct mic_bootparam __iomem *bootparam;
603 INIT_WORK(&mdrv->hotplug_work, mic_hotplug_devices);
604 mic_scan_devices(mdrv, !REMOVE_DEVICES);
606 config_db = mic_next_card_db();
607 virtio_config_cookie = mic_request_card_irq(mic_extint_handler, NULL,
608 "virtio_config_intr", mdrv,
610 if (IS_ERR(virtio_config_cookie)) {
611 rc = PTR_ERR(virtio_config_cookie);
615 bootparam = mdrv->dp;
616 iowrite8(config_db, &bootparam->h2c_config_db);
623 * Uninit function for virtio
625 void mic_devices_uninit(struct mic_driver *mdrv)
627 struct mic_bootparam __iomem *bootparam = mdrv->dp;
628 iowrite8(-1, &bootparam->h2c_config_db);
629 mic_free_card_irq(virtio_config_cookie, mdrv);
630 flush_work(&mdrv->hotplug_work);
631 mic_scan_devices(mdrv, REMOVE_DEVICES);